Skip to content

Commit

Permalink
Refactor codec to leverage backwards_codecs (#2546)
Browse files Browse the repository at this point in the history
Refactors codec management to move codecs that are read only into a new
package, backwards codecs. Also, remove codecs that can no longer be
used in 3.0 (everything before 9.1.

With this, I remove the enum structure of KNNCodecVersion, as there is
really only one codec that can be used to actively write at a time.

Signed-off-by: John Mazanec <jmazane@amazon.com>
  • Loading branch information
jmazanec15 authored Feb 19, 2025
1 parent f9d8b53 commit edcbe31
Show file tree
Hide file tree
Showing 33 changed files with 203 additions and 907 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Maintenance
* Update package name to fix compilation issue [#2513](https://github.com/opensearch-project/k-NN/pull/2513)
### Refactoring
* Refactor codec to leverage backwards_codecs [#2546](https://github.com/opensearch-project/k-NN/pull/2546)

## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.19...2.x)
### Features
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,31 +12,37 @@
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.knn.index.codec.KNN80Codec.KNN80CompoundFormat;
import org.opensearch.knn.index.codec.KNN80Codec.KNN80DocValuesFormat;
import org.opensearch.knn.index.codec.KNN9120Codec.DerivedSourceStoredFieldsFormat;
import org.opensearch.knn.index.codec.KNNCodecVersion;
import org.opensearch.knn.index.codec.KNNFormatFacade;
import org.opensearch.knn.index.codec.KNN9120Codec.KNN9120PerFieldKnnVectorsFormat;
import org.opensearch.knn.index.codec.derivedsource.DerivedSourceReadersSupplier;

import java.util.Optional;

/**
* KNN Codec that wraps the Lucene Codec which is part of Lucene 10.0.1
*/

public class KNN10010Codec extends FilterCodec {

private static final KNNCodecVersion VERSION = KNNCodecVersion.V_10_01_0;
private final KNNFormatFacade knnFormatFacade;
private static final String NAME = "KNN10010Codec";
public static final Codec DEFAULT_DELEGATE = new Lucene101Codec();
private static final PerFieldKnnVectorsFormat DEFAULT_KNN_VECTOR_FORMAT = new KNN9120PerFieldKnnVectorsFormat(Optional.empty());

private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat;
private final StoredFieldsFormat storedFieldsFormat;

private final MapperService mapperService;

/**
* No arg constructor that uses Lucene99 as the delegate
* No arg constructor that uses Lucene101Codec as the delegate
*/
public KNN10010Codec() {
this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldKnnVectorsFormat(), null);
this(DEFAULT_DELEGATE, DEFAULT_KNN_VECTOR_FORMAT, null);
}

/**
Expand All @@ -47,22 +53,21 @@ public KNN10010Codec() {
* @param knnVectorsFormat per field format for KnnVector
*/
@Builder
protected KNN10010Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat, MapperService mapperService) {
super(VERSION.getCodecName(), delegate);
knnFormatFacade = VERSION.getKnnFormatFacadeSupplier().apply(delegate);
public KNN10010Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat, MapperService mapperService) {
super(NAME, delegate);
perFieldKnnVectorsFormat = knnVectorsFormat;
this.mapperService = mapperService;
this.storedFieldsFormat = getStoredFieldsFormat();
}

@Override
public DocValuesFormat docValuesFormat() {
return knnFormatFacade.docValuesFormat();
return new KNN80DocValuesFormat(delegate.docValuesFormat());
}

@Override
public CompoundFormat compoundFormat() {
return knnFormatFacade.compoundFormat();
return new KNN80CompoundFormat(delegate.compoundFormat());
}

@Override
Expand Down

This file was deleted.

This file was deleted.

Loading

0 comments on commit edcbe31

Please sign in to comment.