Audit and fix up serialize_subset uses
Fixes https://github.com/harfbuzz/harfbuzz/issues/4362
diff --git a/src/OT/Color/COLR/COLR.hh b/src/OT/Color/COLR/COLR.hh
index 6591bb4..744774e 100644
--- a/src/OT/Color/COLR/COLR.hh
+++ b/src/OT/Color/COLR/COLR.hh
@@ -1425,8 +1425,8 @@
auto *out = c->serializer->embed (this);
if (unlikely (!out)) return_trace (false);
- if (!out->src.serialize_subset (c, src, this, instancer)) return_trace (false);
- return_trace (out->backdrop.serialize_subset (c, backdrop, this, instancer));
+ return_trace (out->src.serialize_subset (c, src, this, instancer) |
+ out->backdrop.serialize_subset (c, backdrop, this, instancer));
}
bool sanitize (hb_sanitize_context_t *c) const
@@ -1898,15 +1898,16 @@
auto *out = c->serializer->start_embed (this);
if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
+ bool ret = false;
for (const auto& _ : + hb_enumerate (*this)
| hb_filter (c->plan->colrv1_layers, hb_first))
{
auto *o = out->serialize_append (c->serializer);
- if (unlikely (!o) || !o->serialize_subset (c, _.second, this, instancer))
- return_trace (false);
+ if (unlikely (!o)) return_trace (false);
+ ret |= o->serialize_subset (c, _.second, this, instancer);
}
- return_trace (true);
+ return_trace (ret);
}
bool sanitize (hb_sanitize_context_t *c) const
diff --git a/src/OT/Layout/GPOS/AnchorMatrix.hh b/src/OT/Layout/GPOS/AnchorMatrix.hh
index bd9b189..37ba791 100644
--- a/src/OT/Layout/GPOS/AnchorMatrix.hh
+++ b/src/OT/Layout/GPOS/AnchorMatrix.hh
@@ -65,14 +65,15 @@
if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
out->rows = num_rows;
+ bool ret = false;
for (const unsigned i : index_iter)
{
auto *offset = c->serializer->embed (matrixZ[i]);
if (!offset) return_trace (false);
- offset->serialize_subset (c, matrixZ[i], this);
+ ret |= offset->serialize_subset (c, matrixZ[i], this);
}
- return_trace (true);
+ return_trace (ret);
}
};
diff --git a/src/OT/Layout/GPOS/CursivePosFormat1.hh b/src/OT/Layout/GPOS/CursivePosFormat1.hh
index 54852aa..0f9aee2 100644
--- a/src/OT/Layout/GPOS/CursivePosFormat1.hh
+++ b/src/OT/Layout/GPOS/CursivePosFormat1.hh
@@ -24,16 +24,15 @@
(src_base+exitAnchor).collect_variation_indices (c);
}
- EntryExitRecord* subset (hb_subset_context_t *c,
- const void *src_base) const
+ bool subset (hb_subset_context_t *c,
+ const void *src_base) const
{
TRACE_SERIALIZE (this);
auto *out = c->serializer->embed (this);
- if (unlikely (!out)) return_trace (nullptr);
+ if (unlikely (!out)) return_trace (false);
- out->entryAnchor.serialize_subset (c, entryAnchor, src_base);
- out->exitAnchor.serialize_subset (c, exitAnchor, src_base);
- return_trace (out);
+ return_trace (out->entryAnchor.serialize_subset (c, entryAnchor, src_base) |
+ out->exitAnchor.serialize_subset (c, exitAnchor, src_base));
}
protected:
diff --git a/src/OT/Layout/GPOS/LigatureArray.hh b/src/OT/Layout/GPOS/LigatureArray.hh
index a2d807c..59cca40 100644
--- a/src/OT/Layout/GPOS/LigatureArray.hh
+++ b/src/OT/Layout/GPOS/LigatureArray.hh
@@ -27,6 +27,7 @@
auto *out = c->serializer->start_embed (this);
if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
+ bool ret = false;
for (const auto _ : + hb_zip (coverage, *this)
| hb_filter (glyphset, hb_first))
{
@@ -38,13 +39,13 @@
+ hb_range (src.rows * class_count)
| hb_filter ([=] (unsigned index) { return klass_mapping->has (index % class_count); })
;
- matrix->serialize_subset (c,
- _.second,
- this,
- src.rows,
- indexes);
+ ret |= matrix->serialize_subset (c,
+ _.second,
+ this,
+ src.rows,
+ indexes);
}
- return_trace (this->len);
+ return_trace (ret);
}
};
diff --git a/src/OT/Layout/GPOS/MarkArray.hh b/src/OT/Layout/GPOS/MarkArray.hh
index 34e2ab8..0887cc1 100644
--- a/src/OT/Layout/GPOS/MarkArray.hh
+++ b/src/OT/Layout/GPOS/MarkArray.hh
@@ -82,10 +82,10 @@
| hb_map (hb_second)
;
+ bool ret = false;
unsigned new_length = 0;
for (const auto& mark_record : mark_iter) {
- if (unlikely (!mark_record.subset (c, this, klass_mapping)))
- return_trace (false);
+ ret |= mark_record.subset (c, this, klass_mapping);
new_length++;
}
@@ -93,7 +93,7 @@
HB_SERIALIZE_ERROR_ARRAY_OVERFLOW)))
return_trace (false);
- return_trace (true);
+ return_trace (ret);
}
};
diff --git a/src/OT/Layout/GPOS/MarkBasePosFormat1.hh b/src/OT/Layout/GPOS/MarkBasePosFormat1.hh
index eb47120..1b8f3c8 100644
--- a/src/OT/Layout/GPOS/MarkBasePosFormat1.hh
+++ b/src/OT/Layout/GPOS/MarkBasePosFormat1.hh
@@ -197,9 +197,10 @@
if (!out->markCoverage.serialize_serialize (c->serializer, new_coverage.iter ()))
return_trace (false);
- out->markArray.serialize_subset (c, markArray, this,
- (this+markCoverage).iter (),
- &klass_mapping);
+ if (unlikely (!out->markArray.serialize_subset (c, markArray, this,
+ (this+markCoverage).iter (),
+ &klass_mapping)))
+ return_trace (false);
unsigned basecount = (this+baseArray).rows;
auto base_iter =
@@ -228,11 +229,9 @@
;
}
- out->baseArray.serialize_subset (c, baseArray, this,
- base_iter.len (),
- base_indexes.iter ());
-
- return_trace (true);
+ return_trace (out->baseArray.serialize_subset (c, baseArray, this,
+ base_iter.len (),
+ base_indexes.iter ()));
}
};
diff --git a/src/OT/Layout/GPOS/MarkLigPosFormat1.hh b/src/OT/Layout/GPOS/MarkLigPosFormat1.hh
index 92e83a0..af8b472 100644
--- a/src/OT/Layout/GPOS/MarkLigPosFormat1.hh
+++ b/src/OT/Layout/GPOS/MarkLigPosFormat1.hh
@@ -195,9 +195,10 @@
if (!out->markCoverage.serialize_serialize (c->serializer, new_mark_coverage))
return_trace (false);
- out->markArray.serialize_subset (c, markArray, this,
- (this+markCoverage).iter (),
- &klass_mapping);
+ if (unlikely (!out->markArray.serialize_subset (c, markArray, this,
+ (this+markCoverage).iter (),
+ &klass_mapping)))
+ return_trace (false);
auto new_ligature_coverage =
+ hb_iter (this + ligatureCoverage)
@@ -208,10 +209,9 @@
if (!out->ligatureCoverage.serialize_serialize (c->serializer, new_ligature_coverage))
return_trace (false);
- out->ligatureArray.serialize_subset (c, ligatureArray, this,
- hb_iter (this+ligatureCoverage), classCount, &klass_mapping);
-
- return_trace (true);
+ return_trace (out->ligatureArray.serialize_subset (c, ligatureArray, this,
+ hb_iter (this+ligatureCoverage),
+ classCount, &klass_mapping));
}
};
diff --git a/src/OT/Layout/GPOS/MarkMarkPosFormat1.hh b/src/OT/Layout/GPOS/MarkMarkPosFormat1.hh
index 72535f4..70cf071 100644
--- a/src/OT/Layout/GPOS/MarkMarkPosFormat1.hh
+++ b/src/OT/Layout/GPOS/MarkMarkPosFormat1.hh
@@ -183,9 +183,10 @@
if (!out->mark1Coverage.serialize_serialize (c->serializer, new_coverage.iter ()))
return_trace (false);
- out->mark1Array.serialize_subset (c, mark1Array, this,
- (this+mark1Coverage).iter (),
- &klass_mapping);
+ if (unlikely (!out->mark1Array.serialize_subset (c, mark1Array, this,
+ (this+mark1Coverage).iter (),
+ &klass_mapping)))
+ return_trace (false);
unsigned mark2count = (this+mark2Array).rows;
auto mark2_iter =
@@ -214,9 +215,10 @@
;
}
- out->mark2Array.serialize_subset (c, mark2Array, this, mark2_iter.len (), mark2_indexes.iter ());
+ return_trace (out->mark2Array.serialize_subset (c, mark2Array, this,
+ mark2_iter.len (),
+ mark2_indexes.iter ()));
- return_trace (true);
}
};
diff --git a/src/OT/Layout/GPOS/MarkRecord.hh b/src/OT/Layout/GPOS/MarkRecord.hh
index a7d489d..3d11c77 100644
--- a/src/OT/Layout/GPOS/MarkRecord.hh
+++ b/src/OT/Layout/GPOS/MarkRecord.hh
@@ -24,17 +24,16 @@
return_trace (c->check_struct (this) && markAnchor.sanitize (c, base));
}
- MarkRecord *subset (hb_subset_context_t *c,
- const void *src_base,
- const hb_map_t *klass_mapping) const
+ bool subset (hb_subset_context_t *c,
+ const void *src_base,
+ const hb_map_t *klass_mapping) const
{
TRACE_SUBSET (this);
auto *out = c->serializer->embed (this);
- if (unlikely (!out)) return_trace (nullptr);
+ if (unlikely (!out)) return_trace (false);
out->klass = klass_mapping->get (klass);
- out->markAnchor.serialize_subset (c, markAnchor, src_base);
- return_trace (out);
+ return_trace (out->markAnchor.serialize_subset (c, markAnchor, src_base));
}
void collect_variation_indices (hb_collect_variation_indices_context_t *c,
diff --git a/src/hb-ot-layout-common.hh b/src/hb-ot-layout-common.hh
index 0831e44..2cc5098 100644
--- a/src/hb-ot-layout-common.hh
+++ b/src/hb-ot-layout-common.hh
@@ -3215,8 +3215,7 @@
if (unlikely (!out)) return_trace (false);
out->featureIndex = c->feature_index_map->get (featureIndex);
- bool ret = out->feature.serialize_subset (c->subset_context, feature, base, c);
- return_trace (ret);
+ return_trace (out->feature.serialize_subset (c->subset_context, feature, base, c));
}
bool sanitize (hb_sanitize_context_t *c, const void *base) const
