Cache frozen FieldType to skip schema validation (#15886)

When indexing, every field in every document has its schema built via
updateDocFieldSchema and validated via assertSameSchema against the
existing FieldInfo. For the common case where a field consistently uses
the same frozen FieldType instance, this work is redundant — a frozen
type is immutable, so its schema contribution is identical every time.

This change caches the frozen FieldType on each PerField and checks
same object instance to detect when the type hasn't changed. When it
matches, schema building and validation are skipped entirely, and
FieldSchema only resets its docID. If a different type is encountered,
the cache is invalidated and the full validation path runs. A
deoptimize path handles multi-valued fields where a later value uses a
different type than earlier values within the same document.

Adds FieldType.isFrozen() to support the optimization, and new tests
covering the fast-path, cache invalidation, deoptimize, cross-segment
validation, and document blocks.

Author: Tim-Brooks

lucene/CHANGES.txt

@@ -72,6 +72,8 @@ Optimizations
 
 * GITHUB#15779: Improve BytesRefHash.add performance by optimize rehash operation (tyronecai)
 
+* GITHUB#15886: Cache frozen FieldType to skip redundant schema validation. (Tim Brooks)
+
 Bug Fixes
 ---------------------
 * GITHUB#15754: Fix HTMLStripCharFilter to prevent tags from incorrectly consuming subsequent

lucene/core/src/java/org/apache/lucene/document/FieldType.java

@@ -96,6 +96,15 @@ public void freeze() {
     this.frozen = true;
   }
 
+  /**
+   * Returns the frozen FieldTypes's state.
+   *
+   * @return <code>true</code> if this FieldType is frozen against future modifications.
+   */
+  public boolean isFrozen() {
+    return frozen;
+  }
+
   /**
    * {@inheritDoc}
    *

lucene/core/src/java/org/apache/lucene/index/IndexingChain.java

@@ -559,8 +559,8 @@ private void finishStoredFields() throws IOException {
   }
 
   void processDocument(int docID, Iterable<? extends IndexableField> document) throws IOException {
-    // number of unique fields by names (collapses multiple field instances by the same name)
-    int fieldCount = 0;
+    // number of unique fields by name which need to be init in segment or full validation
+    int fieldsNeedInitOrValidate = 0;
     int indexedFieldCount = 0; // number of unique fields indexed with postings
     long fieldGen = nextFieldGen++;
     int docFieldIdx = 0;
@@ -577,40 +577,40 @@ void processDocument(int docID, Iterable<? extends IndexableField> document) thr
       // 1st pass over doc fields – verify that doc schema matches the index schema
       // build schema for each unique doc field
       for (IndexableField field : document) {
-        IndexableFieldType fieldType = field.fieldType();
+        final String fieldName = field.name();
+        final IndexableFieldType fieldType = field.fieldType();
         final boolean isReserved = field.getClass() == ReservedField.class;
         PerField pf =
             getOrAddPerField(
-                field.name(), false
+                fieldName, false
                 /* we never add reserved fields during indexing should be done during DWPT setup*/ );
         if (pf.reserved != isReserved) {
           throw new IllegalArgumentException(
-              "\""
-                  + field.name()
-                  + "\" is a reserved field and should not be added to any document");
+              "\"" + fieldName + "\" is a reserved field and should not be added to any document");
         }
         if (pf.fieldGen != fieldGen) { // first time we see this field in this document
-          fields[fieldCount++] = pf;
           pf.fieldGen = fieldGen;
-          pf.reset(docID);
+          pf.reset(docID, fieldType);
+          if (pf.validatedFrozenFieldType == null) {
+            fields[fieldsNeedInitOrValidate++] = pf;
+          }
+        } else if (pf.multiValueForcesDeoptimize(fieldType)) {
+          // Multi-valued field with a different field type than the cached frozen type.
+          // Drop the validated frozen field type to force the validation path.
+          pf.validatedFrozenFieldType = null;
+          fields[fieldsNeedInitOrValidate++] = pf;
         }
         if (docFieldIdx >= docFields.length) oversizeDocFields();
         docFields[docFieldIdx++] = pf;
-        updateDocFieldSchema(field.name(), pf.schema, fieldType);
-      }
-      // For each field, if it's the first time we see this field in this segment,
-      // initialize its FieldInfo.
-      // If we have already seen this field, verify that its schema
-      // within the current doc matches its schema in the index.
-      for (int i = 0; i < fieldCount; i++) {
-        PerField pf = fields[i];
-        if (pf.fieldInfo == null) {
-          initializeFieldInfo(pf);
-        } else {
-          pf.schema.assertSameSchema(pf.fieldInfo);
+        if (pf.validatedFrozenFieldType == null) {
+          updateDocFieldSchema(fieldName, pf.schema, fieldType);
         }
       }
 
+      if (fieldsNeedInitOrValidate > 0) {
+        initAndValidateFields(fieldsNeedInitOrValidate);
+      }
+
       // 2nd pass over doc fields – index each field
       // also count the number of unique fields indexed with postings
       docFieldIdx = 0;
@@ -641,6 +641,22 @@ void processDocument(int docID, Iterable<? extends IndexableField> document) thr
     }
   }
 
+  private void initAndValidateFields(int fieldCount) throws IOException {
+    // For each field, if it's the first time we see this field in this segment,
+    // initialize its FieldInfo.
+    // If we have already seen this field, verify that its schema
+    // within the current doc matches its schema in the index.
+    for (int i = 0; i < fieldCount; i++) {
+      PerField pf = fields[i];
+      if (pf.fieldInfo == null) {
+        initializeFieldInfo(pf);
+        pf.trySetValidatedFrozenFieldType();
+      } else {
+        pf.schema.assertSameSchema(pf.fieldInfo);
+      }
+    }
+  }
+
   private void oversizeDocFields() {
     PerField[] newDocFields =
         new PerField
@@ -1108,6 +1124,14 @@ private final class PerField implements Comparable<PerField> {
     private final Analyzer analyzer;
     private boolean first; // first in a document
 
+    /**
+     * Allows IndexingChain to skip schema validation if fields keep using the same frozen field
+     * type
+     */
+    private FieldType validatedFrozenFieldType;
+
+    private IndexableFieldType candidateFieldType;
+
     PerField(
         String fieldName,
         int indexCreatedVersionMajor,
@@ -1125,9 +1149,33 @@ private final class PerField implements Comparable<PerField> {
       this.reserved = reserved;
     }
 
-    void reset(int docId) {
+    void reset(int docId, IndexableFieldType fieldType) {
       first = true;
-      schema.reset(docId);
+      if (fieldInfo == null) {
+        // The first time we encounter this field in a segment propose a frozen field to optimize
+        // the validation step. This will be promoted in trySetValidatedFrozenFieldType if it is
+        // frozen and valid.
+        candidateFieldType = fieldType;
+      }
+      if (fieldType == validatedFrozenFieldType) {
+        schema.resetJustDocId(docId);
+      } else {
+        // Encountered new FieldType. Deoptimize the schema validation skip.
+        validatedFrozenFieldType = null;
+        schema.reset(docId);
+      }
+    }
+
+    boolean multiValueForcesDeoptimize(IndexableFieldType fieldType) {
+      return validatedFrozenFieldType != null && fieldType != validatedFrozenFieldType;
+    }
+
+    void trySetValidatedFrozenFieldType() {
+      assert fieldInfo != null;
+      if (candidateFieldType instanceof FieldType ft && ft.isFrozen()) {
+        validatedFrozenFieldType = ft;
+      }
+      candidateFieldType = null;
     }
 
     void setFieldInfo(FieldInfo fieldInfo) {
@@ -1545,8 +1593,12 @@ void setVectors(
       }
     }
 
-    void reset(int doc) {
+    void resetJustDocId(int doc) {
       docID = doc;
+    }
+
+    void reset(int doc) {
+      resetJustDocId(doc);
       omitNorms = false;
       storeTermVector = false;
       indexOptions = IndexOptions.NONE;