refactor command line feature and document it

aigenpipeline-commandline/src/main/java/net/stoerr/ai/aigenpipeline/commandline/AIGenPipeline.java

@@ -88,20 +88,20 @@ public class AIGenPipeline {
     protected List<AIInOut> hintFiles = new ArrayList<>();
 
     public static void main(String[] args) throws IOException {
-        if (args.length == 2 && args[0].equals("-rc")) {
-            processCommandFile(args);
+        if (args.length == 1) {
+            processCommandFile(args[0]);
         } else {
             new AIGenPipeline().run(args);
         }
     }
 
     /**
-     * -rc <file>           Read command lines from the given file. Empty lines separate individual command lines.
+     * Read command lines from the given file. Empty lines separate individual command lines.
      * Lines starting with a # are ignored (comments).
      * This saves the startup time when calling the tool multiple times. Incompatible to all other options.
      */
-    protected static void processCommandFile(String[] args) throws IOException {
-        File cmdfile = new File(args[1]);
+    protected static void processCommandFile(String cmdfilepath) throws IOException {
+        File cmdfile = new File(cmdfilepath);
         if (!cmdfile.exists() || !cmdfile.isFile() || !cmdfile.canRead()) {
             ERR.println("Cannot read command file " + cmdfile.getAbsolutePath());
             System.exit(1);
@@ -111,20 +111,27 @@ protected static void processCommandFile(String[] args) throws IOException {
             StringBuffer cmd = new StringBuffer();
             while (scanner.hasNextLine()) {
                 String line = scanner.nextLine().trim();
-                if (line.startsWith("#")) continue;
+                if (line.trim().startsWith("#")) continue;
                 if (line.trim().isEmpty()) {
-                    new AIGenPipeline().run(cmd.toString().trim().split("\\s+"));
+                    runWithCommandLine(cmd.toString());
                     cmd.setLength(0);
                 } else {
                     cmd.append(" ").append(line.trim());
                 }
             }
             if (!cmd.toString().trim().isEmpty()) {
-                new AIGenPipeline().run(cmd.toString().trim().split("\\s+"));
+                runWithCommandLine(cmd.toString());
             }
         }
     }
 
+    protected static void runWithCommandLine(String cmdline) throws IOException {
+        ERR.println("Processing command line: ");
+        ERR.println(cmdline.toString().trim());
+        new AIGenPipeline().run(cmdline.trim().split("\\s+"));
+        ERR.println();
+    }
+
     protected void run(String[] args) throws IOException {
         try {
             readArguments(args, rootDir);

aigenpipeline-commandline/src/main/resources/aigencmdline/usage.txt

@@ -1,5 +1,13 @@
 Usage:
 aigenpipeline [options] [<input_files>...]
+or
+ai-gen-pipeline <command_file>
+
+The AIGenPipeline tool generates content using an AI based on a prompt and input files.
+It can also update or improve existing content, and it only calls the AI if the input or prompt files have changed.
+
+If it's called with a command file, it reads a number of command lines from that file. Empty lines separate individual command lines.
+Lines starting with a # are ignored (comments). This saves the startup time when calling the tool multiple times.
 
 Options:
 
@@ -14,9 +22,6 @@ Options:
     -n, --dry-run            Enable dry-run mode, where the tool will only print to stderr what it would do without 
                              actually calling the AI or writing any files.
     -v, --verbose            Enable verbose output to stderr, providing more details about the process.
-    -rc <file>               Read command lines from the given file. Empty lines separate individual command lines.
-                             Lines starting with a # are ignored (comments).
-                             This saves the startup time when calling the tool multiple times. Incompatible to all other options.
 
   Input / outputs:
     -o, --output <file>      Specify the output file where the generated content will be written.

bin/aigenpipeline

@@ -3,9 +3,24 @@
 scriptdir="$(dirname $(readlink -f $0))"
 
 # find the jar file. Try first according to project layout, then in scriptdir itself.
-jarFile="$(ls -1tr $scriptdir/../aigenpipeline-commandline/target/aigenpipeline-commandline*.jar | egrep -v 'sources|javadoc' | tail -n 1)"
+jarFile="$(ls -1tr $scriptdir/../aigenpipeline-commandline/target/aigenpipeline-commandline*.jar | egrep -v 'sources|javadoc')"
+
+# abort if there are several jar files
+if [ $(echo "$jarFile" | wc -l) -gt 1 ]; then
+  echo "Cannot execute: multiple jar files found in $scriptdir/../aigenpipeline-commandline/target" >&2
+  echo "$jarFile" >&2
+  exit 1
+fi
+
 if [ -z "$jarFile" ]; then
-  jarFile="$(ls -1tr $scriptdir/aigenpipeline-commandline*.jar | egrep -v 'sources|javadoc' | tail -n 1)"
+  jarFile="$(ls -1tr $scriptdir/aigenpipeline-commandline*.jar | egrep -v 'sources|javadoc')"
+fi
+
+# abort if there are several jar files
+if [ $(echo "$jarFile" | wc -l) -gt 1 ]; then
+  echo "Cannot execute: multiple jar files found in $scriptdir" >&2
+  echo "$jarFile" >&2
+  exit 1
 fi
 
 if [ -z "$jarFile" ]; then

examples/differentialReTranslation/generate-new.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env ../../bin/aigenpipeline
+-m gpt-4o-mini -p 0dialogelements.prompt README.md -o dialogelements.txt
+
+-m gpt-4o-mini -p 1html.prompt README.md dialogelements.txt -o differentialReTranslation.html
+
+-m gpt-4o-mini -p 2css.prompt README.md differentialReTranslation.html -o differentialReTranslation.css
+
+-m gpt-4o -p 3js.prompt README.md dialogelements.txt requests.jsonl -o differentialReTranslation.js
+
+-m gpt-4o-mini -p 4examplejs.prompt README.md dialogelements.txt examples.txt -o differentialReTranslationExamples.js

src/site/markdown/index.md

@@ -163,6 +163,28 @@ command line arguments. Thus, the later override the earlier one. Explicitly giv
 processed at the point where the argument occurs when processing the command line arguments. The option `-cp` /
 `--configprint` gives an overview of the used files / sources of configuration.
 
+## Command files
+
+While the startup time of aigenpipeline is low in comparison to the actual LLM calls, it can still hurt if there
+are many files to process and most AI calls can be skipped because there were no changes in inputs or prompts.
+Thus, if you just give one file as argument, it'll be read s command file containing a number of command lines
+that are executed in sequence.
+Those contain a number of command lines that are executed in sequence. Empty lines separate individual command lines,
+and lines starting with a # are ignored (comments). For example:
+
+```
+#!/usr/bin/env ../../bin/aigenpipeline
+-m gpt-4o-mini -p 0dialogelements.prompt README.md -o dialogelements.txt
+
+# command lines are separated by empty lines, and comment lines starting with # are ignored
+-m gpt-4o-mini -p 1html.prompt README.md dialogelements.txt -o differentialReTranslation.html
+
+# Each command line can be split over several lines if convenient.
+-m gpt-4o-mini -p 2css.prompt 
+README.md differentialReTranslation.html 
+-o differentialReTranslation.css
+```
+
 ## Other features
 
 If you are not satisfied with the result, the tool can also be used to ask the AI for clarification: ask a question
@@ -199,6 +221,14 @@ You can either:
 ```
 Usage:
 aigenpipeline [options] [<input_files>...]
+or
+ai-gen-pipeline <command_file>
+
+The AIGenPipeline tool generates content using an AI based on a prompt and input files.
+It can also update or improve existing content, and it only calls the AI if the input or prompt files have changed.
+
+If it's called with a command file, it reads a number of command lines from that file. Empty lines separate individual command lines.
+Lines starting with a # are ignored (comments). This saves the startup time when calling the tool multiple times.
 
 Options: