Skip to content

Commit

Permalink
Add local CLI tests
Browse files Browse the repository at this point in the history
  • Loading branch information
R0Wi committed Feb 12, 2025
1 parent 3f0ebe6 commit ed62fed
Show file tree
Hide file tree
Showing 11 changed files with 365 additions and 210 deletions.
15 changes: 13 additions & 2 deletions .github/workflows/phpunit-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,9 @@ jobs:
php-versions: ['8.3']
databases: ['mysql']
server-versions: ['stable31']
backend: ['remote', 'local'] # Do not change these names, they're used in the integration tests

name: php-integrationtests${{ matrix.php-versions }}-${{ matrix.databases }}
name: php-integrationtests-${{ matrix.backend }}-${{ matrix.php-versions }}-${{ matrix.databases }}

steps:
- name: Checkout server
Expand Down Expand Up @@ -75,6 +76,12 @@ jobs:
extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_sqlite, gd, zip, imagick
coverage: none

- name: Install ocrmypdf
if: matrix.backend == 'local'
run: |
sudo apt-get update && sudo apt-get install -y ocrmypdf
ocrmypdf --version
- name: Install composer dependencies
working-directory: apps/${{ env.APP_NAME }}
run: composer i
Expand All @@ -93,12 +100,14 @@ jobs:
- name: Checkout AppApi
uses: actions/checkout@v4
if: matrix.backend == 'remote'
with:
repository: nextcloud/app_api
ref: ${{ matrix.server-versions }}
path: apps/app_api

- name: Set up AppApi/ExApp infrastructure
if: matrix.backend == 'remote'
run: |
./occ app:enable app_api
./occ app_api:daemon:register local_docker "docker-socket-proxy" \
Expand All @@ -110,10 +119,12 @@ jobs:
- name: PHPUnit
working-directory: apps/${{ env.APP_NAME }}
env:
GITHUB_MATRIX_BACKEND: ${{ matrix.backend }}
run: make php-integrationtest

- name: Write OCR Backend logs to file
if: failure()
if: failure() && matrix.backend == 'remote'
run: |
docker logs nc_app_workflow_ocr_backend > data/ocr_backend.log
Expand Down
2 changes: 2 additions & 0 deletions lib/Helper/SidecarFileAccessor.php
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ public function getOrCreateSidecarFile() {
$this->sidecarFilePath = $this->tempManager->getTemporaryFile('sidecar');
if (!$this->sidecarFilePath) {
$this->logger->warning('Could not create temporary sidecar file');
} else if (!is_writable($this->sidecarFilePath)) {
$this->logger->warning('Temporary sidecar file is not writable');
}
}
return $this->sidecarFilePath;
Expand Down
11 changes: 3 additions & 8 deletions lib/OcrProcessors/CommandLineUtils.php
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@

namespace OCA\WorkflowOcr\OcrProcessors;

use OCA\WorkflowOcr\Helper\ISidecarFileAccessor;
use OCA\WorkflowOcr\Model\GlobalSettings;
use OCA\WorkflowOcr\Model\WorkflowSettings;
use OCA\WorkflowOcr\Service\IOcrBackendInfoService;
Expand All @@ -38,13 +37,12 @@ class CommandLineUtils implements ICommandLineUtils {
];

public function __construct(
private ISidecarFileAccessor $sidecarFileAccessor,
private IOcrBackendInfoService $ocrBackendInfoService,
private LoggerInterface $logger,
) {
}

public function getCommandlineArgs(WorkflowSettings $settings, GlobalSettings $globalSettings, array $additionalCommandlineArgs = []): string {
public function getCommandlineArgs(WorkflowSettings $settings, GlobalSettings $globalSettings, string $sidecarFile = null, array $additionalCommandlineArgs = []): string {
$isLocalExecution = !$this->ocrBackendInfoService->isRemoteBackend();

// Default setting is quiet
Expand Down Expand Up @@ -75,12 +73,9 @@ public function getCommandlineArgs(WorkflowSettings $settings, GlobalSettings $g
$args[] = '--jobs ' . $processorCount;
}

if ($isLocalExecution) {
if ($isLocalExecution && $sidecarFile !== null) {
// Save recognized text in tempfile
$sidecarFilePath = $this->sidecarFileAccessor->getOrCreateSidecarFile();
if ($sidecarFilePath) {
$args[] = '--sidecar ' . $sidecarFilePath;
}
$args[] = '--sidecar ' . $sidecarFile;
}

$resultArgs = array_filter(array_merge(
Expand Down
2 changes: 1 addition & 1 deletion lib/OcrProcessors/ICommandLineUtils.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,5 @@
use OCA\WorkflowOcr\Model\WorkflowSettings;

interface ICommandLineUtils {
public function getCommandlineArgs(WorkflowSettings $settings, GlobalSettings $globalSettings, array $additionalCommandlineArgs = []): string;
public function getCommandlineArgs(WorkflowSettings $settings, GlobalSettings $globalSettings, string $sidecarFile = null, array $additionalCommandlineArgs = []): string;
}
5 changes: 3 additions & 2 deletions lib/OcrProcessors/Local/OcrMyPdfBasedProcessor.php
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ public function __construct(

public function ocrFile(File $file, WorkflowSettings $settings, GlobalSettings $globalSettings): OcrProcessorResult {
$additionalCommandlineArgs = $this->getAdditionalCommandlineArgs($settings, $globalSettings);
$commandStr = 'ocrmypdf ' . $this->commandLineUtils->getCommandlineArgs($settings, $globalSettings, $additionalCommandlineArgs) . ' - - || exit $? ; cat';
$sidecarFile = $this->sidecarFileAccessor->getOrCreateSidecarFile();
$commandStr = 'ocrmypdf ' . $this->commandLineUtils->getCommandlineArgs($settings, $globalSettings, $sidecarFile, $additionalCommandlineArgs) . ' - - || exit $? ; cat';

$inputFileContent = $file->getContent();

Expand Down Expand Up @@ -82,7 +83,7 @@ public function ocrFile(File $file, WorkflowSettings $settings, GlobalSettings $
$recognizedText = $this->sidecarFileAccessor->getSidecarFileContent();

if (!$recognizedText) {
$this->logger->info('Temporary sidecar file at \'{path}\' was empty', ['path' => $this->sidecarFileAccessor->getOrCreateSidecarFile()]);
$this->logger->info('Temporary sidecar file at \'{path}\' was empty', ['path' => $sidecarFile]);
}

$this->logger->debug('OCR processing was successful');
Expand Down
215 changes: 215 additions & 0 deletions tests/Integration/BackendTestBase.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
<?php

declare(strict_types=1);

/**
* @copyright Copyright (c) 2025 Robin Windey <ro.windey@gmail.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

namespace OCA\WorkflowOcr\Tests\Integration;

use CurlHandle;
use DomainException;
use OCA\WorkflowEngine\Helper\ScopeContext;
use OCA\WorkflowEngine\Manager;
use OCA\WorkflowOcr\AppInfo\Application;
use OCA\WorkflowOcr\BackgroundJobs\ProcessFileJob;
use OCA\WorkflowOcr\Operation;
use OCA\WorkflowOcr\Service\IOcrBackendInfoService;
use OCA\WorkflowOcr\Wrapper\CommandWrapper;
use OCP\AppFramework\App;
use OCP\BackgroundJob\IJobList;
use OCP\IConfig;
use OCP\WorkflowEngine\IManager;
use Psr\Container\ContainerInterface;
use Test\TestCase;

abstract class BackendTestBase extends TestCase {
private IConfig $config;
private ScopeContext $context;
private Manager $workflowEngineManager;
private $oldLogLevel;
private $operationClass = Operation::class;
private array $uploadedFiles = [];

protected ContainerInterface $container;

protected function setUp(): void {
parent::setUp();

$app = new App(Application::APP_NAME);
$this->container = $app->getContainer();
$this->config = $this->container->get(IConfig::class);
$this->workflowEngineManager = $this->container->get(Manager::class);
$this->context = new ScopeContext(IManager::SCOPE_ADMIN);

$this->setNextcloudLogLevel();
$this->deleteOperation();
}

protected function tearDown(): void {
$this->restoreNextcloudLogLevel();
$this->deleteTestFilesIfExist();
$this->deleteOperation();
parent::tearDown();
}

protected function setNextcloudLogLevel() : void {
$this->oldLogLevel = $this->config->getSystemValue('loglevel', 3);
$this->config->setSystemValue('loglevel', 0);
}

protected function restoreNextcloudLogLevel() : void {
$this->config->setSystemValue('loglevel', $this->oldLogLevel);
}

protected function runOcrBackgroundJob() {
/** @var IJoblist */
$jobList = $this->container->get(IJobList::class);
$job = $jobList->getNext(false, [ProcessFileJob::class]);
$this->assertNotNull($job, 'Expected one background job');
$job->start($jobList);
}

protected function checkOcrBackendServiceInstalled() : bool {
$ocrBackendInfoService = $this->container->get(IOcrBackendInfoService::class);
return $ocrBackendInfoService->isRemoteBackend();
}

protected function checkOcrMyPdfInstalled() : bool {
$command = new CommandWrapper();
$command->setCommand('ocrmypdf --version')->execute();
return $command->getExitCode() === 0;
}

protected function addOperation(string $mimeType) {
// NOTE :: we're creating the workflow operation via
// REST API because if we'd use the manager directly, we'd
// face some issues because of caching etc (test ist running
// in another process than webserver ...)
$url = $this->getNextcloudOcsApiUrl() . 'apps/workflowengine/api/v1/workflows/global?format=json';
$json = '
{
"id":-1,
"class":"' . str_replace('\\', '\\\\', $this->operationClass) . '",
"entity":"OCA\\\\WorkflowEngine\\\\Entity\\\\File",
"events":["\\\\OCP\\\\Files::postCreate"],
"name":"",
"checks":[
{
"class":"OCA\\\\WorkflowEngine\\\\Check\\\\FileMimeType",
"operator":"is",
"value":"' . $mimeType . '",
"invalid":false
}
],
"operation":"",
"valid":true
}';

$ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $json);
curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
'OCS-APIREQUEST: true'
]);

$this->executeCurl($ch);
}

private function deleteOperation() {
$operations = $this->workflowEngineManager->getOperations($this->operationClass, $this->context);
foreach ($operations as $operation) {
try {
$this->workflowEngineManager->deleteOperation($operation['id'], $this->context);
} catch (DomainException) {
// ignore
}
}

}

protected function uploadTestFile(string $testFile) {
$localFile = __DIR__ . "/testdata/" . $testFile;
$this->uploadedFiles[] = $localFile;
$file = fopen($localFile, 'r');

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, $this->getNextcloudWebdavUrl() . basename($localFile));
curl_setopt($ch, CURLOPT_PUT, true);
curl_setopt($ch, CURLOPT_INFILE, $file);
curl_setopt($ch, CURLOPT_INFILESIZE, filesize($localFile));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$this->executeCurl($ch);
fclose($file);
}

private function deleteTestFilesIfExist() {
foreach ($this->uploadedFiles as $localFile) {
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, $this->getNextcloudWebdavUrl() . basename($localFile));
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$this->executeCurl($ch, [404]);
}
$this->uploadedFiles = [];
}

private function executeCurl(CurlHandle $ch, array $allowedNonSuccessResponseCodes = []) : string|bool {
curl_setopt($ch, CURLOPT_USERPWD, $this->getNextcloudCredentials());

$result = curl_exec($ch);

if (curl_errno($ch)) {
$this->fail('cURL Error: ' . curl_error($ch));
}

$responseCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($responseCode >= 400 && !in_array($responseCode, $allowedNonSuccessResponseCodes)) {
$responseBody = curl_multi_getcontent($ch);
$this->fail('cURL HTTP Error ' . $responseCode . ': ' . $responseBody);
}

curl_close($ch);

return $result;
}

private function getNextcloudWebdavUrl() : string {
$port = getenv('NEXTCLOUD_PORT') ?: '80';
$user = getenv('NEXTCLOUD_USER') ?: 'admin';
return 'http://localhost:' . $port . '/remote.php/dav/files/' . $user . '/';
}

private function getNextcloudOcsApiUrl() : string {
$port = getenv('NEXTCLOUD_PORT') ?: '80';
return 'http://localhost:' . $port . '/ocs/v2.php/';
}

private function getNextcloudCredentials() : string {
$user = getenv('NEXTCLOUD_USER') ?: 'admin';
$pass = getenv('NEXTCLOUD_PASS') ?: 'admin';
return $user . ':' . $pass;
}
}
Loading

0 comments on commit ed62fed

Please sign in to comment.