Integrate Log10 plugin: Add opt-in reporting and restructure (#318)

* Integrate pytest_log10_managed_evaluation into main package

- Move plugin files to src/pytest_log10_managed_evaluation/
- Update pyproject.toml to include plugin
- Remove separate plugin pyproject.toml

* add option to enable log10 plugin

* Update README for opt-in Log10 reporting

- Add --log10 option and INI setting
- Update usage instructions and examples
- Clarify new opt-in behavior for Log10 reporting

* restructure the dir to src/log10/pytest_log10_managed_evaluation

* minor update:
- update poetry.lock
- revert gh workflow test.yml to use --all-extras

.github/workflows/test.yml

@@ -63,7 +63,7 @@ jobs:
           cache: "poetry"
           architecture: 'x64'
       - name: Install dependencies
-        run: poetry install --extras "autofeedback_icl litellm langchain gemini mistralai together mosaicml google-generativeai lamini cli"
+        run: poetry install  --all-extras
 
       - name: Run cli tests
         run: poetry run pytest -vv tests/test_cli.py

pyproject.toml

@@ -41,6 +41,8 @@ anthropic = "<1"
 requests = "^2.31.0"
 python-dotenv = "^1.0.0"
 backoff = "^2.2.1"
+pytest = ">=8.0.0"
+pytest-metadata = ">=1.0.0"
 langchain = {version = "<0.2.0", optional = true}
 magentic = {version = ">=0.17.0", optional = true, markers = "python_version >= '3.10'"}
 litellm = {version = "^1.41.12", optional = true}
@@ -55,8 +57,6 @@ click = {version = "^8.1.7", optional = true}
 rich = {version = "^13.7.1", optional = true}
 tabulate = {version = "^0.9.0", optional = true}
 pandas = {version = ">=2", optional = true}
-# pytest = {version = ">=8.0.0", optional = true}
-# pytest-metadata = {version = ">=1.0.0", optional = true}
 
 [tool.poetry.extras]
 autofeedback_icl = ["magentic"]
@@ -69,11 +69,9 @@ mosaicml = ["mosaicml-cli"]
 google-generativeai = ["google-generativeai"]
 lamini = ["lamini"]
 cli = ["click", "rich", "tabulate", "pandas"]
-pytest = ["pytest_log10_managed_evaluation"]
 
-[tool.poetry.dependencies.pytest_log10_managed_evaluation]
-path = "src/pytest_log10_managed_evaluation"
-optional = true
+[tool.poetry.plugins."pytest11"]
+log10_managed_evaluation = "log10.pytest_log10_managed_evaluation.plugin"
 
 [tool.ruff]
 # Never enforce `E501` (line length violations).

src/pytest_log10_managed_evaluation/README.md → src/log10/pytest_log10_managed_evaluation/README.md

@@ -5,33 +5,47 @@ A pytest plugin for managing evaluation in Log10 platform.
 ## Installation
 After [configuring the Log10 environment variables](https://docs.log10.io/observability/advanced/logging#configuration),
 ```bash
-pip install 'log10-io[pytest]'
+pip install log10-io
 ```
 
 ## Options
 
 | Option | Type | Description |
 |--------|------|-------------|
+| `--log10` | Command-line | Enable Log10 managed evaluation reporting |
 | `--eval-session-name` | Command-line | Set name for the evaluation session |
 | `--local` | Command-line | Run pytest locally without showing the session in Log10. A JSON report file will be saved locally. |
 | `--pretty-print` | Command-line | Pretty-print the JSON report with indentation. |
+| `log10` | INI | Enable Log10 managed evaluation reporting. Set in `pytest.ini` or `setup.cfg`. |
 | `eval_session_name` | INI | Set name for the evaluation session. Set in `pytest.ini` or `setup.cfg`. |
 
 ## Usage
 
-Once installed, the plugin is automatically enabled when you run pytest. Simply execute your tests as you normally would, e.g.:
+To enable the Log10 managed evaluation reporting, you need to use the `--log10` option or set `log10 = true` in your pytest configuration file. Once enabled, execute your tests as you normally would:
+
 ```bash
-pytest tests
+pytest tests --log10
 ```
 
 This will run your tests and upload the results to the Log10 platform for managed evaluation.
 
+### Enabling Log10 Reporting in Configuration
+
+You can enable Log10 reporting by default in your pytest configuration file:
+
+Example in `pytest.ini`:
+
+```ini
+[pytest]
+log10 = true
+```
+
 ### Running Tests Locally
 
 If you prefer to run tests locally without uploading results to Log10, use the `--local` option:
 
 ```bash
-pytest tests --local
+pytest tests --log10 --local
 ```
 
 When using the `--local` option, a JSON report will be generated and saved in the `.pytest_log10_eval_reports` folder in your project directory. This is useful for:
@@ -45,7 +59,7 @@ This helps in organizing and identifying specific test runs in the Log10 platfor
 To assign a custom name to your evaluation session, use the `--eval-session-name` option:
 
 ```bash
-pytest tests --eval-session-name <your-test-session-name>
+pytest tests --log10 --eval-session-name <your-test-session-name>
 ```
 
 Use `eval_session_name` in INI file
@@ -64,7 +78,7 @@ eval_session_name = <your-test-session-name>
 For improved readability of local JSON reports, use the `--pretty-print` option:
 
 ```bash
-pytest tests --local --pretty-print
+pytest tests --log10 --local --pretty-print
 ```
 
 This will format the JSON report with proper indentation, making it easier to read and analyze.

src/pytest_log10_managed_evaluation/pytest_log10_managed_evaluation/plugin.py → src/log10/pytest_log10_managed_evaluation/plugin.py

@@ -416,6 +416,12 @@ def json_metadata(request):
 
 def pytest_addoption(parser):
     group = parser.getgroup("jsonreport", "reporting test results as JSON")
+    group.addoption(
+        "--log10",
+        action="store_true",
+        default=False,
+        help="Enable Log10 managed evaluation reporting",
+    )
     group.addoption(
         "--pretty-print",
         action="store_true",
@@ -432,12 +438,22 @@ def pytest_addoption(parser):
         "--eval-session-name", action="store", default=None, help="Name for the evaluation session in JSON report"
     )
     parser.addini("eval_session_name", "Name for the evaluation session in JSON report", default=None)
+    parser.addini(
+        "log10",
+        "Enable Log10 managed evaluation reporting",
+        type="bool",
+        default=False,
+    )
 
 
 def pytest_configure(config):
     if config.option.collectonly:
         return
 
+    log10_enabled = config.getoption("log10") or config.getini("log10")
+    if not log10_enabled:
+        return
+
     if hasattr(config, "workerinput"):
         Plugin = JSONReportWorker
     else: