From 81ac143ea41ef7762e8786904b90ab5284bf7dac Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Mon, 9 Dec 2024 22:09:07 +0100 Subject: [PATCH 01/12] Embedding for Mistral and an example with Mistral Embed and PGVector --- examples/mistral-embedding-example/go.mod | 20 ++ examples/mistral-embedding-example/go.sum | 248 ++++++++++++++++++ .../mistral-embedding-example.go | 131 +++++++++ go.mod | 10 +- go.sum | 4 +- llms/mistral/mistralembed.go | 34 +++ llms/mistral/mistralembed_test.go | 88 +++++++ 7 files changed, 530 insertions(+), 5 deletions(-) create mode 100644 examples/mistral-embedding-example/go.mod create mode 100644 examples/mistral-embedding-example/go.sum create mode 100644 examples/mistral-embedding-example/mistral-embedding-example.go create mode 100644 llms/mistral/mistralembed.go create mode 100644 llms/mistral/mistralembed_test.go diff --git a/examples/mistral-embedding-example/go.mod b/examples/mistral-embedding-example/go.mod new file mode 100644 index 000000000..6de5d92b5 --- /dev/null +++ b/examples/mistral-embedding-example/go.mod @@ -0,0 +1,20 @@ +module github.com/mathiasb/langchaingo/examples/mistral-embedding-example + +go 1.23.4 + +require github.com/tmc/langchaingo v0.12.1 + +replace github.com/tmc/langchaingo => ../../ + +require ( + github.com/dlclark/regexp2 v1.10.0 // indirect + github.com/gage-technologies/mistral-go v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/pgvector/pgvector-go v0.1.1 // indirect + github.com/pkoukk/tiktoken-go v0.1.6 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/text v0.15.0 // indirect +) diff --git a/examples/mistral-embedding-example/go.sum b/examples/mistral-embedding-example/go.sum new file mode 100644 index 000000000..497e8dda2 --- /dev/null +++ b/examples/mistral-embedding-example/go.sum @@ -0,0 +1,248 @@ +cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= +cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= +cloud.google.com/go/ai v0.7.0 h1:P6+b5p4gXlza5E+u7uvcgYlzZ7103ACg70YdZeC6oGE= +cloud.google.com/go/ai v0.7.0/go.mod h1:7ozuEcraovh4ABsPbrec3o4LmFl9HigNI3D5haxYeQo= +cloud.google.com/go/aiplatform v1.68.0 h1:EPPqgHDJpBZKRvv+OsB3cr0jYz3EL2pZ+802rBPcG8U= +cloud.google.com/go/aiplatform v1.68.0/go.mod h1:105MFA3svHjC3Oazl7yjXAmIR89LKhRAeNdnDKJczME= +cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw= +cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= +cloud.google.com/go/vertexai v0.12.0 h1:zTadEo/CtsoyRXNx3uGCncoWAP1H2HakGqwznt+iMo8= +cloud.google.com/go/vertexai v0.12.0/go.mod h1:8u+d0TsvBfAAd2x5R6GMgbYhsLgo3J7lmP4bR8g2ig8= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= +github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/containerd/containerd v1.7.15 h1:afEHXdil9iAm03BmhjzKyXnnEBtjaLJefdU7DV0IFes= +github.com/containerd/containerd v1.7.15/go.mod h1:ISzRRTMF8EXNpJlTzyr2XMhN+j9K302C21/+cr3kUnY= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= +github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/gage-technologies/mistral-go v1.1.0 h1:POv1wM9jA/9OBXGV2YdPi9Y/h09+MjCbUF+9hRYlVUI= +github.com/gage-technologies/mistral-go v1.1.0/go.mod h1:tF++Xt7U975GcLlzhrjSQb8l/x+PrriO9QEdsgm9l28= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0= +github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/generative-ai-go v0.15.1 h1:n8aQUpvhPOlGVuM2DRkJ2jvx04zpp42B778AROJa+pQ= +github.com/google/generative-ai-go v0.15.1/go.mod h1:AAucpWZjXsDKhQYWvCYuP6d0yB1kX998pJlOW1rAesw= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= +github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pgvector/pgvector-go v0.1.1 h1:kqJigGctFnlWvskUiYIvJRNwUtQl/aMSUZVs0YWQe+g= +github.com/pgvector/pgvector-go v0.1.1/go.mod h1:wLJgD/ODkdtd2LJK4l6evHXTuG+8PxymYAVomKHOWac= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= +github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/testcontainers/testcontainers-go v0.31.0 h1:W0VwIhcEVhRflwL9as3dhY6jXjVCA27AkmbnZ+UTh3U= +github.com/testcontainers/testcontainers-go v0.31.0/go.mod h1:D2lAoA0zUFiSY+eAflqK5mcUx/A5hrrORaEQrd0SefI= +github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0 h1:isAwFS3KNKRbJMbWv+wolWqOFUECmjYZ+sIRZCIBc/E= +github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0/go.mod h1:ZNYY8vumNCEG9YI59A9d6/YaMY49uwRhmeU563EzFGw= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/pgdialect v1.1.12 h1:m/CM1UfOkoBTglGO5CUTKnIKKOApOYxkcP2qn0F9tJk= +github.com/uptrace/bun/dialect/pgdialect v1.1.12/go.mod h1:Ij6WIxQILxLlL2frUBxUBOZJtLElD2QQNDcu/PWDHTc= +github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibggiLdTqjTtfHc1w= +github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.starlark.net v0.0.0-20230302034142-4b1e35fe2254 h1:Ss6D3hLXTM0KobyBYEAygXzFfGcjnmfEJOBgSbemCtg= +go.starlark.net v0.0.0-20230302034142-4b1e35fe2254/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +google.golang.org/api v0.183.0 h1:PNMeRDwo1pJdgNcFQ9GstuLe/noWKIc89pRWRLMvLwE= +google.golang.org/api v0.183.0/go.mod h1:q43adC5/pHoSZTx5h2mSmdF7NcyfW9JuDyIOJAgS9ZQ= +google.golang.org/genproto v0.0.0-20240528184218-531527333157 h1:u7WMYrIrVvs0TF5yaKwKNbcJyySYf+HAIFXxWltJOXE= +google.golang.org/genproto v0.0.0-20240528184218-531527333157/go.mod h1:ubQlAQnzejB8uZzszhrTCU2Fyp6Vi7ZE5nn0c3W8+qQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/examples/mistral-embedding-example/mistral-embedding-example.go b/examples/mistral-embedding-example/mistral-embedding-example.go new file mode 100644 index 000000000..a85381c0b --- /dev/null +++ b/examples/mistral-embedding-example/mistral-embedding-example.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "github.com/tmc/langchaingo/embeddings" + "github.com/tmc/langchaingo/llms/mistral" + "github.com/tmc/langchaingo/schema" + "github.com/tmc/langchaingo/vectorstores" + "github.com/tmc/langchaingo/vectorstores/pgvector" +) + +func main() { + var dsn string + flag.StringVar(&dsn, "dsn", "", "PGvector connection string") + flag.Parse() + model, err := mistral.New() + if err != nil { + panic(err) + } + + e, err := embeddings.NewEmbedder(model) + + if err != nil { + panic(err) + } + + // Create a new pgvector store. + ctx := context.Background() + store, err := pgvector.New( + ctx, + pgvector.WithConnectionURL(dsn), + pgvector.WithEmbedder(e), + ) + if err != nil { + log.Fatal("pgvector.New", err) + } + + // Add documents to the pgvector store. + _, err = store.AddDocuments(context.Background(), []schema.Document{ + { + PageContent: "Tokyo", + Metadata: map[string]any{ + "population": 38, + "area": 2190, + }, + }, + { + PageContent: "Paris", + Metadata: map[string]any{ + "population": 11, + "area": 105, + }, + }, + { + PageContent: "London", + Metadata: map[string]any{ + "population": 9.5, + "area": 1572, + }, + }, + { + PageContent: "Santiago", + Metadata: map[string]any{ + "population": 6.9, + "area": 641, + }, + }, + { + PageContent: "Buenos Aires", + Metadata: map[string]any{ + "population": 15.5, + "area": 203, + }, + }, + { + PageContent: "Rio de Janeiro", + Metadata: map[string]any{ + "population": 13.7, + "area": 1200, + }, + }, + { + PageContent: "Sao Paulo", + Metadata: map[string]any{ + "population": 22.6, + "area": 1523, + }, + }, + }) + if err != nil { + log.Fatal("store.AddDocuments:\n", err) + } + time.Sleep(1 * time.Second) + + // Search for similar documents. + docs, err := store.SimilaritySearch(ctx, "japan", 1) + if err != nil { + log.Fatal("store.SimilaritySearch1:\n", err) + } + fmt.Println("store.SimilaritySearch1:\n", docs) + + time.Sleep(2 * time.Second) // Don't trigger cloudflare + + // Search for similar documents using score threshold. + docs, err = store.SimilaritySearch(ctx, "only cities in south america", 3, vectorstores.WithScoreThreshold(0.50)) + if err != nil { + log.Fatal("store.SimilaritySearch2:\n", err) + } + fmt.Println("store.SimilaritySearch2:\n", docs) + + time.Sleep(3 * time.Second) // Don't trigger cloudflare + + // Search for similar documents using score threshold and metadata filter. + // Metadata filter for pgvector only supports key-value pairs for now. + filter := map[string]any{"area": "1523"} // Sao Paulo + + docs, err = store.SimilaritySearch(ctx, "only cities in south america", + 3, + vectorstores.WithScoreThreshold(0.50), + vectorstores.WithFilters(filter), + ) + if err != nil { + log.Fatal("store.SimilaritySearch3:\n", err) + } + fmt.Println("store.SimilaritySearch3:\n", docs) +} diff --git a/go.mod b/go.mod index f3270f4c2..3cf01483e 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,11 @@ -module github.com/tmc/langchaingo +module github.com/mathiasb/langchaingo go 1.22.0 +require github.com/tmc/langchaingo v0.12.1 + +replace github.com/tmc/langchaingo => ./ + require ( github.com/google/uuid v1.6.0 github.com/stretchr/testify v1.9.0 @@ -158,7 +162,6 @@ require ( gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82 // indirect gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 // indirect gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f // indirect - go.mongodb.org/mongo-driver/v2 v2.0.0-beta1 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect @@ -196,7 +199,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.8.1 github.com/cohere-ai/tokenizer v1.1.2 github.com/fatih/color v1.17.0 - github.com/gage-technologies/mistral-go v1.0.0 + github.com/gage-technologies/mistral-go v1.1.0 github.com/getzep/zep-go v1.0.4 github.com/go-openapi/strfmt v0.21.3 github.com/go-sql-driver/mysql v1.7.1 @@ -220,6 +223,7 @@ require ( github.com/weaviate/weaviate-go-client/v4 v4.13.1 gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a go.mongodb.org/mongo-driver v1.14.0 + go.mongodb.org/mongo-driver/v2 v2.0.0-beta1 go.starlark.net v0.0.0-20230302034142-4b1e35fe2254 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/tools v0.14.0 diff --git a/go.sum b/go.sum index 1579d1c52..837ae4cdb 100644 --- a/go.sum +++ b/go.sum @@ -196,8 +196,8 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gage-technologies/mistral-go v1.0.0 h1:Hwk0uJO+Iq4kMX/EwbfGRUq9zkO36w7HZ/g53N4N73A= -github.com/gage-technologies/mistral-go v1.0.0/go.mod h1:tF++Xt7U975GcLlzhrjSQb8l/x+PrriO9QEdsgm9l28= +github.com/gage-technologies/mistral-go v1.1.0 h1:POv1wM9jA/9OBXGV2YdPi9Y/h09+MjCbUF+9hRYlVUI= +github.com/gage-technologies/mistral-go v1.1.0/go.mod h1:tF++Xt7U975GcLlzhrjSQb8l/x+PrriO9QEdsgm9l28= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= diff --git a/llms/mistral/mistralembed.go b/llms/mistral/mistralembed.go new file mode 100644 index 000000000..f9dcf8109 --- /dev/null +++ b/llms/mistral/mistralembed.go @@ -0,0 +1,34 @@ +package mistral + +import ( + "context" + "errors" +) + +func ConvertFloat64ToFloat32(input []float64) []float32 { + // Create a slice with the same length as the input. + output := make([]float32, len(input)) + + // Iterate over the input slice and convert each element. + for i, v := range input { + output[i] = float32(v) + } + + return output +} + +// CreateEmbedding implements the embeddings.EmbedderClient interface and creates embeddings for the given input texts. +func (m *Model) CreateEmbedding(ctx context.Context, inputTexts []string) ([][]float32, error) { + embsRes, err := m.client.Embeddings("mistral-embed", inputTexts) + if err != nil { + return nil, errors.New("failed to create embeddings: " + err.Error()) + } + var allEmbds [][]float32 + for _, embs := range embsRes.Data { + if len(embs.Embedding) == 0 { + return nil, errors.New("empty embeddings") + } + allEmbds = append(allEmbds, ConvertFloat64ToFloat32(embs.Embedding)) + } + return allEmbds, nil +} diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go new file mode 100644 index 000000000..86aaa7a53 --- /dev/null +++ b/llms/mistral/mistralembed_test.go @@ -0,0 +1,88 @@ +package mistral_test + +import ( + "context" + "math/rand" + "os" + "testing" + + "github.com/tmc/langchaingo/embeddings" + sdk "github.com/tmc/langchaingo/llms/mistral" +) + +func TestConvertFloat64ToFloat32(t *testing.T) { + + // Test case 1: Empty slice + input := []float64{} + output := sdk.ConvertFloat64ToFloat32(input) + if len(output) != 0 { + t.Errorf("Expected output length to be 0, but got %d", len(output)) + } + + // Test case 2: Single element slice + input = []float64{3.14} + output = sdk.ConvertFloat64ToFloat32(input) + if len(output) != 1 || output[0] != float32(3.14) { + t.Errorf("Expected output to be [3.14], but got %v", output) + } + + // Test case 3: Multiple element slice + input = []float64{1.23, 4.56, 7.89} + output = sdk.ConvertFloat64ToFloat32(input) + if len(output) != 3 || output[0] != float32(1.23) || output[1] != float32(4.56) || output[2] != float32(7.89) { + t.Errorf("Expected output to be [1.23 4.56 7.89], but got %v", output) + } + + // Test case 4: Large random slice + input = make([]float64, 1000) + r := rand.New(rand.NewSource(0)) + for i := range input { + input[i] = r.Float64() + } + output = sdk.ConvertFloat64ToFloat32(input) + if len(output) != len(input) { + t.Errorf("Expected output length to be %d, but got %d", len(input), len(output)) + } + for i, v := range input { + if float32(v) != output[i] { + t.Errorf("Expected output[%d] to be %f, but got %f", i, float32(v), output[i]) + } + } +} + +func TestMistralEmbed(t *testing.T) { + envVar := "MISTRAL_API_KEY" + + // Get the value of the environment variable + value := os.Getenv(envVar) + + // Check if it is set (non-empty) + if value == "" { + t.Logf("Environment variable %s is not set, so skipping the test", envVar) + return + } + + model, err := sdk.New() + if err != nil { + panic(err) + } + + e, err := embeddings.NewEmbedder(model) + + if err != nil { + panic(err) + } + + aaa, err := e.EmbedDocuments(context.Background(), []string{"Hello world"}) + if err != nil { + panic(err) + } + t.Logf("aaa EQ: %v\n", aaa) + + bbb, err := e.EmbedQuery(context.Background(), "Hello world") + if err != nil { + panic(err) + } + + t.Logf("bbb EQ: %v\n", bbb) +} From f5ce2417e101ef965dcd81865ba1e68a886c0b9b Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 28 Dec 2024 09:38:17 +0100 Subject: [PATCH 02/12] updated package name in go.mod files and go mod tidy to be ready for PR --- examples/mistral-embedding-example/go.mod | 8 ++-- examples/mistral-embedding-example/go.sum | 50 ++++++++++++----------- go.mod | 6 +-- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/examples/mistral-embedding-example/go.mod b/examples/mistral-embedding-example/go.mod index 6de5d92b5..60f4e997f 100644 --- a/examples/mistral-embedding-example/go.mod +++ b/examples/mistral-embedding-example/go.mod @@ -1,10 +1,10 @@ -module github.com/mathiasb/langchaingo/examples/mistral-embedding-example +module github.com/tmc/langchaingo/examples/mistral-embedding-example -go 1.23.4 +go 1.22.0 -require github.com/tmc/langchaingo v0.12.1 +toolchain go1.22.1 -replace github.com/tmc/langchaingo => ../../ +require github.com/tmc/langchaingo v0.1.12 require ( github.com/dlclark/regexp2 v1.10.0 // indirect diff --git a/examples/mistral-embedding-example/go.sum b/examples/mistral-embedding-example/go.sum index 497e8dda2..6f71dacd2 100644 --- a/examples/mistral-embedding-example/go.sum +++ b/examples/mistral-embedding-example/go.sum @@ -1,21 +1,21 @@ -cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= -cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= -cloud.google.com/go/ai v0.7.0 h1:P6+b5p4gXlza5E+u7uvcgYlzZ7103ACg70YdZeC6oGE= -cloud.google.com/go/ai v0.7.0/go.mod h1:7ozuEcraovh4ABsPbrec3o4LmFl9HigNI3D5haxYeQo= -cloud.google.com/go/aiplatform v1.68.0 h1:EPPqgHDJpBZKRvv+OsB3cr0jYz3EL2pZ+802rBPcG8U= -cloud.google.com/go/aiplatform v1.68.0/go.mod h1:105MFA3svHjC3Oazl7yjXAmIR89LKhRAeNdnDKJczME= -cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw= -cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= +cloud.google.com/go v0.113.0 h1:g3C70mn3lWfckKBiCVsAshabrDg01pQ0pnX1MNtnMkA= +cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= +cloud.google.com/go/ai v0.6.0 h1:QWjb2UoaM15e51IMeLuIUFyWxooKOKDb66Mk47zZ2/g= +cloud.google.com/go/ai v0.6.0/go.mod h1:6/mrRq6aJdK7MZH76ZvcMpESiAiha5aRvurmroiOrgI= +cloud.google.com/go/aiplatform v1.67.0 h1:YWeqD4BjYwrmY4fa+isGcw0P81lJ3dKVxbWxdBchoiU= +cloud.google.com/go/aiplatform v1.67.0/go.mod h1:s/sJ6btBEr6bKnrNWdK9ZgHCvwbZNdP90b3DDtxxw+Y= +cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg= +cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= -cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= +cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= -cloud.google.com/go/vertexai v0.12.0 h1:zTadEo/CtsoyRXNx3uGCncoWAP1H2HakGqwznt+iMo8= -cloud.google.com/go/vertexai v0.12.0/go.mod h1:8u+d0TsvBfAAd2x5R6GMgbYhsLgo3J7lmP4bR8g2ig8= +cloud.google.com/go/vertexai v0.10.0 h1:k157bLrtyajGtAAZnqdEn8lwFlUTG3BgHc7kvWbP/3s= +cloud.google.com/go/vertexai v0.10.0/go.mod h1:w/Zb22QvOVvxx5CGM4fPzH3WA6gwUkId9juA7pigzFI= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= @@ -75,8 +75,8 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/generative-ai-go v0.15.1 h1:n8aQUpvhPOlGVuM2DRkJ2jvx04zpp42B778AROJa+pQ= -github.com/google/generative-ai-go v0.15.1/go.mod h1:AAucpWZjXsDKhQYWvCYuP6d0yB1kX998pJlOW1rAesw= +github.com/google/generative-ai-go v0.14.0 h1:2GwFKXui9LmG+PukQwYk9KpJUIemmQ9NJ46BV9VIw38= +github.com/google/generative-ai-go v0.14.0/go.mod h1:hOzbW3cB5hRV2x05McOwJS4GsqSluYwejjk5tSfb6YY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= @@ -170,6 +170,8 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tmc/langchaingo v0.1.12 h1:yXwSu54f3b1IKw0jJ5/DWu+qFVH1NBblwC0xddBzGJE= +github.com/tmc/langchaingo v0.1.12/go.mod h1:cd62xD6h+ouk8k/QQFhOsjRYBSA1JJ5UVKXSIgm7Ni4= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= @@ -212,8 +214,8 @@ golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= @@ -224,14 +226,14 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -google.golang.org/api v0.183.0 h1:PNMeRDwo1pJdgNcFQ9GstuLe/noWKIc89pRWRLMvLwE= -google.golang.org/api v0.183.0/go.mod h1:q43adC5/pHoSZTx5h2mSmdF7NcyfW9JuDyIOJAgS9ZQ= -google.golang.org/genproto v0.0.0-20240528184218-531527333157 h1:u7WMYrIrVvs0TF5yaKwKNbcJyySYf+HAIFXxWltJOXE= -google.golang.org/genproto v0.0.0-20240528184218-531527333157/go.mod h1:ubQlAQnzejB8uZzszhrTCU2Fyp6Vi7ZE5nn0c3W8+qQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/api v0.180.0 h1:M2D87Yo0rGBPWpo1orwfCLehUUL6E7/TYe5gvMQWDh4= +google.golang.org/api v0.180.0/go.mod h1:51AiyoEg1MJPSZ9zvklA8VnRILPXxn1iVen9v25XHAE= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= +google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 h1:W5Xj/70xIA4x60O/IFyXivR5MGqblAb8R3w26pnD6No= +google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434 h1:umK/Ey0QEzurTNlsV3R+MfxHAb78HCEX/IkuR+zH4WQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= diff --git a/go.mod b/go.mod index 3cf01483e..4260300fc 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,7 @@ -module github.com/mathiasb/langchaingo +module github.com/tmc/langchaingo go 1.22.0 -require github.com/tmc/langchaingo v0.12.1 - -replace github.com/tmc/langchaingo => ./ - require ( github.com/google/uuid v1.6.0 github.com/stretchr/testify v1.9.0 From b51bcb97b7dc0820db319a5de944d9bd266a567c Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 28 Dec 2024 10:26:51 +0100 Subject: [PATCH 03/12] fix CI lint checks --- llms/mistral/mistralembed.go | 8 ++++---- llms/mistral/mistralembed_test.go | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/llms/mistral/mistralembed.go b/llms/mistral/mistralembed.go index f9dcf8109..033548f1a 100644 --- a/llms/mistral/mistralembed.go +++ b/llms/mistral/mistralembed.go @@ -18,17 +18,17 @@ func ConvertFloat64ToFloat32(input []float64) []float32 { } // CreateEmbedding implements the embeddings.EmbedderClient interface and creates embeddings for the given input texts. -func (m *Model) CreateEmbedding(ctx context.Context, inputTexts []string) ([][]float32, error) { +func (m *Model) CreateEmbedding(_ context.Context, inputTexts []string) ([][]float32, error) { embsRes, err := m.client.Embeddings("mistral-embed", inputTexts) if err != nil { return nil, errors.New("failed to create embeddings: " + err.Error()) } - var allEmbds [][]float32 - for _, embs := range embsRes.Data { + var allEmbds = make([][]float32, len(embsRes.Data)) + for i, embs := range embsRes.Data { if len(embs.Embedding) == 0 { return nil, errors.New("empty embeddings") } - allEmbds = append(allEmbds, ConvertFloat64ToFloat32(embs.Embedding)) + allEmbds[i] = ConvertFloat64ToFloat32(embs.Embedding) } return allEmbds, nil } diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go index 86aaa7a53..c13b5fe48 100644 --- a/llms/mistral/mistralembed_test.go +++ b/llms/mistral/mistralembed_test.go @@ -2,7 +2,8 @@ package mistral_test import ( "context" - "math/rand" + "crypto/rand" + "math/big" "os" "testing" @@ -34,10 +35,13 @@ func TestConvertFloat64ToFloat32(t *testing.T) { } // Test case 4: Large random slice - input = make([]float64, 1000) - r := rand.New(rand.NewSource(0)) + input = make([]float64, 1_000_000) + r, err := rand.Int(rand.Reader, big.NewInt(42)) + if err != nil { + panic(err) + } for i := range input { - input[i] = r.Float64() + input[i] = float64(r.Int64()) } output = sdk.ConvertFloat64ToFloat32(input) if len(output) != len(input) { From 315def00b9836054ad9caffa64fd279c7d8fb767 Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 28 Dec 2024 11:05:06 +0100 Subject: [PATCH 04/12] Example is using the new local copy of the Mistral Model which implements CreateEmbedding and the embeddings.EmbedderClient interface --- examples/mistral-embedding-example/go.mod | 2 + examples/mistral-embedding-example/go.sum | 50 +++++++++++------------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/examples/mistral-embedding-example/go.mod b/examples/mistral-embedding-example/go.mod index 60f4e997f..01d1fe27e 100644 --- a/examples/mistral-embedding-example/go.mod +++ b/examples/mistral-embedding-example/go.mod @@ -6,6 +6,8 @@ toolchain go1.22.1 require github.com/tmc/langchaingo v0.1.12 +replace github.com/tmc/langchaingo => ../../ + require ( github.com/dlclark/regexp2 v1.10.0 // indirect github.com/gage-technologies/mistral-go v1.1.0 // indirect diff --git a/examples/mistral-embedding-example/go.sum b/examples/mistral-embedding-example/go.sum index 6f71dacd2..497e8dda2 100644 --- a/examples/mistral-embedding-example/go.sum +++ b/examples/mistral-embedding-example/go.sum @@ -1,21 +1,21 @@ -cloud.google.com/go v0.113.0 h1:g3C70mn3lWfckKBiCVsAshabrDg01pQ0pnX1MNtnMkA= -cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= -cloud.google.com/go/ai v0.6.0 h1:QWjb2UoaM15e51IMeLuIUFyWxooKOKDb66Mk47zZ2/g= -cloud.google.com/go/ai v0.6.0/go.mod h1:6/mrRq6aJdK7MZH76ZvcMpESiAiha5aRvurmroiOrgI= -cloud.google.com/go/aiplatform v1.67.0 h1:YWeqD4BjYwrmY4fa+isGcw0P81lJ3dKVxbWxdBchoiU= -cloud.google.com/go/aiplatform v1.67.0/go.mod h1:s/sJ6btBEr6bKnrNWdK9ZgHCvwbZNdP90b3DDtxxw+Y= -cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg= -cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= +cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= +cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= +cloud.google.com/go/ai v0.7.0 h1:P6+b5p4gXlza5E+u7uvcgYlzZ7103ACg70YdZeC6oGE= +cloud.google.com/go/ai v0.7.0/go.mod h1:7ozuEcraovh4ABsPbrec3o4LmFl9HigNI3D5haxYeQo= +cloud.google.com/go/aiplatform v1.68.0 h1:EPPqgHDJpBZKRvv+OsB3cr0jYz3EL2pZ+802rBPcG8U= +cloud.google.com/go/aiplatform v1.68.0/go.mod h1:105MFA3svHjC3Oazl7yjXAmIR89LKhRAeNdnDKJczME= +cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw= +cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= -cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= -cloud.google.com/go/vertexai v0.10.0 h1:k157bLrtyajGtAAZnqdEn8lwFlUTG3BgHc7kvWbP/3s= -cloud.google.com/go/vertexai v0.10.0/go.mod h1:w/Zb22QvOVvxx5CGM4fPzH3WA6gwUkId9juA7pigzFI= +cloud.google.com/go/vertexai v0.12.0 h1:zTadEo/CtsoyRXNx3uGCncoWAP1H2HakGqwznt+iMo8= +cloud.google.com/go/vertexai v0.12.0/go.mod h1:8u+d0TsvBfAAd2x5R6GMgbYhsLgo3J7lmP4bR8g2ig8= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= @@ -75,8 +75,8 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/generative-ai-go v0.14.0 h1:2GwFKXui9LmG+PukQwYk9KpJUIemmQ9NJ46BV9VIw38= -github.com/google/generative-ai-go v0.14.0/go.mod h1:hOzbW3cB5hRV2x05McOwJS4GsqSluYwejjk5tSfb6YY= +github.com/google/generative-ai-go v0.15.1 h1:n8aQUpvhPOlGVuM2DRkJ2jvx04zpp42B778AROJa+pQ= +github.com/google/generative-ai-go v0.15.1/go.mod h1:AAucpWZjXsDKhQYWvCYuP6d0yB1kX998pJlOW1rAesw= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= @@ -170,8 +170,6 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/tmc/langchaingo v0.1.12 h1:yXwSu54f3b1IKw0jJ5/DWu+qFVH1NBblwC0xddBzGJE= -github.com/tmc/langchaingo v0.1.12/go.mod h1:cd62xD6h+ouk8k/QQFhOsjRYBSA1JJ5UVKXSIgm7Ni4= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= @@ -214,8 +212,8 @@ golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= @@ -226,14 +224,14 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -google.golang.org/api v0.180.0 h1:M2D87Yo0rGBPWpo1orwfCLehUUL6E7/TYe5gvMQWDh4= -google.golang.org/api v0.180.0/go.mod h1:51AiyoEg1MJPSZ9zvklA8VnRILPXxn1iVen9v25XHAE= -google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= -google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= -google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 h1:W5Xj/70xIA4x60O/IFyXivR5MGqblAb8R3w26pnD6No= -google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434 h1:umK/Ey0QEzurTNlsV3R+MfxHAb78HCEX/IkuR+zH4WQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= +google.golang.org/api v0.183.0 h1:PNMeRDwo1pJdgNcFQ9GstuLe/noWKIc89pRWRLMvLwE= +google.golang.org/api v0.183.0/go.mod h1:q43adC5/pHoSZTx5h2mSmdF7NcyfW9JuDyIOJAgS9ZQ= +google.golang.org/genproto v0.0.0-20240528184218-531527333157 h1:u7WMYrIrVvs0TF5yaKwKNbcJyySYf+HAIFXxWltJOXE= +google.golang.org/genproto v0.0.0-20240528184218-531527333157/go.mod h1:ubQlAQnzejB8uZzszhrTCU2Fyp6Vi7ZE5nn0c3W8+qQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= From 56c7c7e88356a60e41a1dc5e8661d671aa3d45b7 Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 28 Dec 2024 11:10:40 +0100 Subject: [PATCH 05/12] Fixed lint errors in test cases --- llms/mistral/mistralembed_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go index c13b5fe48..e8fb018a5 100644 --- a/llms/mistral/mistralembed_test.go +++ b/llms/mistral/mistralembed_test.go @@ -11,8 +11,8 @@ import ( sdk "github.com/tmc/langchaingo/llms/mistral" ) +// TestConvertFloat64ToFloat32 tests the ConvertFloat64ToFloat32 function func TestConvertFloat64ToFloat32(t *testing.T) { - // Test case 1: Empty slice input := []float64{} output := sdk.ConvertFloat64ToFloat32(input) @@ -55,6 +55,7 @@ func TestConvertFloat64ToFloat32(t *testing.T) { } func TestMistralEmbed(t *testing.T) { + t.Parallel() envVar := "MISTRAL_API_KEY" // Get the value of the environment variable From 822bda950ad7fa3563989fd308702572647fe202 Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 28 Dec 2024 11:17:14 +0100 Subject: [PATCH 06/12] Fixed more lint errors in test cases --- llms/mistral/mistralembed_test.go | 78 +++++++++++++++---------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go index e8fb018a5..b794bf3fd 100644 --- a/llms/mistral/mistralembed_test.go +++ b/llms/mistral/mistralembed_test.go @@ -2,8 +2,6 @@ package mistral_test import ( "context" - "crypto/rand" - "math/big" "os" "testing" @@ -11,46 +9,48 @@ import ( sdk "github.com/tmc/langchaingo/llms/mistral" ) -// TestConvertFloat64ToFloat32 tests the ConvertFloat64ToFloat32 function +// TestConvertFloat64ToFloat32 tests the ConvertFloat64ToFloat32 function using table-driven tests. func TestConvertFloat64ToFloat32(t *testing.T) { - // Test case 1: Empty slice - input := []float64{} - output := sdk.ConvertFloat64ToFloat32(input) - if len(output) != 0 { - t.Errorf("Expected output length to be 0, but got %d", len(output)) + tests := []struct { + name string + input []float64 + expected []float32 + }{ + { + name: "empty slice", + input: []float64{}, + expected: []float32{}, + }, + { + name: "single element", + input: []float64{3.14}, + expected: []float32{3.14}, + }, + { + name: "multiple elements", + input: []float64{1.23, 4.56, 7.89}, + expected: []float32{1.23, 4.56, 7.89}, + }, + { + name: "zero values", + input: []float64{0.0, 0.0, 0.0}, + expected: []float32{0.0, 0.0, 0.0}, + }, } - // Test case 2: Single element slice - input = []float64{3.14} - output = sdk.ConvertFloat64ToFloat32(input) - if len(output) != 1 || output[0] != float32(3.14) { - t.Errorf("Expected output to be [3.14], but got %v", output) - } - - // Test case 3: Multiple element slice - input = []float64{1.23, 4.56, 7.89} - output = sdk.ConvertFloat64ToFloat32(input) - if len(output) != 3 || output[0] != float32(1.23) || output[1] != float32(4.56) || output[2] != float32(7.89) { - t.Errorf("Expected output to be [1.23 4.56 7.89], but got %v", output) - } - - // Test case 4: Large random slice - input = make([]float64, 1_000_000) - r, err := rand.Int(rand.Reader, big.NewInt(42)) - if err != nil { - panic(err) - } - for i := range input { - input[i] = float64(r.Int64()) - } - output = sdk.ConvertFloat64ToFloat32(input) - if len(output) != len(input) { - t.Errorf("Expected output length to be %d, but got %d", len(input), len(output)) - } - for i, v := range input { - if float32(v) != output[i] { - t.Errorf("Expected output[%d] to be %f, but got %f", i, float32(v), output[i]) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + output := sdk.ConvertFloat64ToFloat32(tt.input) + if len(output) != len(tt.expected) { + t.Errorf("length mismatch: got %d, want %d", len(output), len(tt.expected)) + return + } + for i := range output { + if output[i] != tt.expected[i] { + t.Errorf("at index %d: got %f, want %f", i, output[i], tt.expected[i]) + } + } + }) } } From 41b2ab1468420bf8b4d9056bad685a5ce8658054 Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 28 Dec 2024 11:20:57 +0100 Subject: [PATCH 07/12] Fixed more lint error Parallel in test cases --- llms/mistral/mistralembed_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go index b794bf3fd..8930ae037 100644 --- a/llms/mistral/mistralembed_test.go +++ b/llms/mistral/mistralembed_test.go @@ -11,6 +11,7 @@ import ( // TestConvertFloat64ToFloat32 tests the ConvertFloat64ToFloat32 function using table-driven tests. func TestConvertFloat64ToFloat32(t *testing.T) { + t.Parallel() tests := []struct { name string input []float64 @@ -40,6 +41,7 @@ func TestConvertFloat64ToFloat32(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() output := sdk.ConvertFloat64ToFloat32(tt.input) if len(output) != len(tt.expected) { t.Errorf("length mismatch: got %d, want %d", len(output), len(tt.expected)) From 50c4eae3d98d39d6ce68a12e24ea99c036aa0811 Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 28 Dec 2024 11:29:53 +0100 Subject: [PATCH 08/12] Fixed more lint errors --- llms/mistral/mistralembed.go | 2 +- llms/mistral/mistralembed_test.go | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/llms/mistral/mistralembed.go b/llms/mistral/mistralembed.go index 033548f1a..5d21e5fa9 100644 --- a/llms/mistral/mistralembed.go +++ b/llms/mistral/mistralembed.go @@ -23,7 +23,7 @@ func (m *Model) CreateEmbedding(_ context.Context, inputTexts []string) ([][]flo if err != nil { return nil, errors.New("failed to create embeddings: " + err.Error()) } - var allEmbds = make([][]float32, len(embsRes.Data)) + allEmbds := make([][]float32, len(embsRes.Data)) for i, embs := range embsRes.Data { if len(embs.Embedding) == 0 { return nil, errors.New("empty embeddings") diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go index 8930ae037..5956c32a7 100644 --- a/llms/mistral/mistralembed_test.go +++ b/llms/mistral/mistralembed_test.go @@ -75,21 +75,19 @@ func TestMistralEmbed(t *testing.T) { } e, err := embeddings.NewEmbedder(model) - if err != nil { panic(err) } - aaa, err := e.EmbedDocuments(context.Background(), []string{"Hello world"}) + docEmbeddings, err := e.EmbedDocuments(context.Background(), []string{"Hello world"}) if err != nil { panic(err) } - t.Logf("aaa EQ: %v\n", aaa) + t.Logf("Document embeddings: %v\n", docEmbeddings) - bbb, err := e.EmbedQuery(context.Background(), "Hello world") + queryEmbedding, err := e.EmbedQuery(context.Background(), "Hello world") if err != nil { panic(err) } - - t.Logf("bbb EQ: %v\n", bbb) + t.Logf("Query embedding: %v\n", queryEmbedding) } From 9a96546619fe04ee6420e3dd2fc1d64deff8dcb2 Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 11 Jan 2025 17:20:22 +0100 Subject: [PATCH 09/12] minor fixes after feedback from @FluffyKebab --- llms/mistral/mistralembed.go | 10 +++++--- llms/mistral/mistralembed_test.go | 39 +++++++++++-------------------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/llms/mistral/mistralembed.go b/llms/mistral/mistralembed.go index 5d21e5fa9..af667310d 100644 --- a/llms/mistral/mistralembed.go +++ b/llms/mistral/mistralembed.go @@ -5,7 +5,11 @@ import ( "errors" ) -func ConvertFloat64ToFloat32(input []float64) []float32 { +var ( + ErrEmptyEmbeddings = errors.New("empty embeddings") +) + +func convertFloat64ToFloat32(input []float64) []float32 { // Create a slice with the same length as the input. output := make([]float32, len(input)) @@ -26,9 +30,9 @@ func (m *Model) CreateEmbedding(_ context.Context, inputTexts []string) ([][]flo allEmbds := make([][]float32, len(embsRes.Data)) for i, embs := range embsRes.Data { if len(embs.Embedding) == 0 { - return nil, errors.New("empty embeddings") + return nil, ErrEmptyEmbeddings } - allEmbds[i] = ConvertFloat64ToFloat32(embs.Embedding) + allEmbds[i] = convertFloat64ToFloat32(embs.Embedding) } return allEmbds, nil } diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go index 5956c32a7..79aba798a 100644 --- a/llms/mistral/mistralembed_test.go +++ b/llms/mistral/mistralembed_test.go @@ -1,12 +1,12 @@ -package mistral_test +package mistral import ( "context" "os" "testing" + "github.com/stretchr/testify/require" "github.com/tmc/langchaingo/embeddings" - sdk "github.com/tmc/langchaingo/llms/mistral" ) // TestConvertFloat64ToFloat32 tests the ConvertFloat64ToFloat32 function using table-driven tests. @@ -42,15 +42,11 @@ func TestConvertFloat64ToFloat32(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - output := sdk.ConvertFloat64ToFloat32(tt.input) - if len(output) != len(tt.expected) { - t.Errorf("length mismatch: got %d, want %d", len(output), len(tt.expected)) - return - } + output := convertFloat64ToFloat32(tt.input) + + require.Equal(t, len(tt.expected), len(output), "length mismatch") for i := range output { - if output[i] != tt.expected[i] { - t.Errorf("at index %d: got %f, want %f", i, output[i], tt.expected[i]) - } + require.Equal(t, tt.expected[i], output[i], "at index %d", i) } }) } @@ -65,29 +61,22 @@ func TestMistralEmbed(t *testing.T) { // Check if it is set (non-empty) if value == "" { - t.Logf("Environment variable %s is not set, so skipping the test", envVar) + t.Skipf("Environment variable %s is not set, so skipping the test", envVar) return } - model, err := sdk.New() + model, err := New() if err != nil { panic(err) } e, err := embeddings.NewEmbedder(model) - if err != nil { - panic(err) - } + require.NoError(t, err) - docEmbeddings, err := e.EmbedDocuments(context.Background(), []string{"Hello world"}) - if err != nil { - panic(err) - } - t.Logf("Document embeddings: %v\n", docEmbeddings) + _, err = e.EmbedDocuments(context.Background(), []string{"Hello world"}) + require.NoError(t, err) + + _, err = e.EmbedQuery(context.Background(), "Hello world") + require.NoError(t, err) - queryEmbedding, err := e.EmbedQuery(context.Background(), "Hello world") - if err != nil { - panic(err) - } - t.Logf("Query embedding: %v\n", queryEmbedding) } From cbce9b520518c463883692886a007763f46a104c Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 11 Jan 2025 17:35:20 +0100 Subject: [PATCH 10/12] linting fixes --- llms/mistral/mistralembed_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go index 79aba798a..139ab4dbf 100644 --- a/llms/mistral/mistralembed_test.go +++ b/llms/mistral/mistralembed_test.go @@ -66,9 +66,7 @@ func TestMistralEmbed(t *testing.T) { } model, err := New() - if err != nil { - panic(err) - } + require.NoError(t, err) e, err := embeddings.NewEmbedder(model) require.NoError(t, err) @@ -79,4 +77,4 @@ func TestMistralEmbed(t *testing.T) { _, err = e.EmbedQuery(context.Background(), "Hello world") require.NoError(t, err) -} +} \ No newline at end of file From 9e0beaa10b052a1231e309a149b917d7a915333f Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 11 Jan 2025 17:46:50 +0100 Subject: [PATCH 11/12] linting fixes --- llms/mistral/mistralembed.go | 4 +--- llms/mistral/mistralembed_test.go | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/llms/mistral/mistralembed.go b/llms/mistral/mistralembed.go index af667310d..f9bb11f18 100644 --- a/llms/mistral/mistralembed.go +++ b/llms/mistral/mistralembed.go @@ -5,9 +5,7 @@ import ( "errors" ) -var ( - ErrEmptyEmbeddings = errors.New("empty embeddings") -) +var ErrEmptyEmbeddings = errors.New("empty embeddings") func convertFloat64ToFloat32(input []float64) []float32 { // Create a slice with the same length as the input. diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go index 139ab4dbf..07be648cb 100644 --- a/llms/mistral/mistralembed_test.go +++ b/llms/mistral/mistralembed_test.go @@ -76,5 +76,4 @@ func TestMistralEmbed(t *testing.T) { _, err = e.EmbedQuery(context.Background(), "Hello world") require.NoError(t, err) - } \ No newline at end of file From c62063eda6fcb0a5eed54ba4c88ecb22692e9567 Mon Sep 17 00:00:00 2001 From: Mathias Bergqvist Date: Sat, 11 Jan 2025 18:14:45 +0100 Subject: [PATCH 12/12] linting fixes --- llms/mistral/mistralembed_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llms/mistral/mistralembed_test.go b/llms/mistral/mistralembed_test.go index 07be648cb..38a278e93 100644 --- a/llms/mistral/mistralembed_test.go +++ b/llms/mistral/mistralembed_test.go @@ -76,4 +76,4 @@ func TestMistralEmbed(t *testing.T) { _, err = e.EmbedQuery(context.Background(), "Hello world") require.NoError(t, err) -} \ No newline at end of file +}