[fix] regex filters

[add] skip-filter

README.md

@@ -8,11 +8,13 @@ fw:directory-list($test:dir,$options)
 ````
 ## Options
 
-* maxDepth: integer
-* showFileInfo: boolean
-* maxFiles: integer
-* include-filter:string @TODO
-* exclude-filter:string @TODO
+* max-depth: integer directory depth to scan, -1 all
+* include-info: boolean add @size and @last-modified attributes to o/p
+* max-files: integer stop scanning after finding this number of files
+* follow-links: boolean
+* include-filter:string regex  applied only to file names
+* exclude-filter:string regex  applied only to file names
+* skip-filter:string regex applied only to directory names
 
 ## Sample output
 ````

dist/doc/file-walker.xqm.xml

@@ -1,6 +1,6 @@
 <xqdoc:xqdoc xmlns:xqdoc="http://www.xqdoc.org/1.0">
   <xqdoc:control>
-    <xqdoc:date>2017-04-10T13:12:52.133+01:00</xqdoc:date>
+    <xqdoc:date>2017-04-10T17:14:45.352+01:00</xqdoc:date>
     <xqdoc:version>1.1</xqdoc:version>
   </xqdoc:control>
   <xqdoc:module type="library">

makjar.jardesc

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="WINDOWS-1252" standalone="no"?>
 <jardesc>
-    <jar path="file-walker/src/main/content/quodatum-files-0.3.9.jar"/>
+    <jar path="file-walker/src/main/content/quodatum-files-0.5.0.jar"/>
     <options buildIfNeeded="true" compress="true" descriptionLocation="/file-walker/makjar.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
     <storedRefactorings deprecationInfo="true" structuralOnly="false"/>
     <selectedProjects/>

package.xml

@@ -92,5 +92,32 @@
     <version num="0.3.9">
       <!-- generated: {fn:current-dateTime()} -->
     </version>
+    <version num="0.4.0">
+      <!-- generated: {fn:current-dateTime()} -->
+    </version>
+    <version num="0.4.1">
+      <!-- generated: {fn:current-dateTime()} -->
+    </version>
+    <version num="0.4.2">
+      <!-- generated: {fn:current-dateTime()} -->
+    </version>
+    <version num="0.4.3">
+      <!-- generated: {fn:current-dateTime()} -->
+    </version>
+    <version num="0.4.4">
+      <!-- generated: {fn:current-dateTime()} -->
+    </version>
+    <version num="0.4.5">
+      <!-- generated: {fn:current-dateTime()} -->
+    </version>
+    <version num="0.4.6">
+      <!-- generated: {fn:current-dateTime()} -->
+    </version>
+    <version num="0.4.7">
+      <!-- generated: {fn:current-dateTime()} -->
+    </version>
+    <version num="0.5.0">
+      <!-- generated: {fn:current-dateTime()} -->
+    </version>
   </pkg>
 </repo>

src/java/com/quodatum/file/Runner.java

@@ -1,6 +1,11 @@
 package com.quodatum.file;
 
-// 
+/*
+ * @author andy bunce
+ * @copyright Quodatum Ltd
+ * @date 2015-2017
+ * @licence Apache 2
+ */
 import java.io.IOException;
 import java.nio.file.FileVisitOption;
 import java.nio.file.Files;
@@ -20,21 +25,26 @@ public class Runner extends QueryModule {
     //@TODO use this
 	public static class RunnerOptions extends Options {
 		/** Max depth of directories to scan. */
-		public static final NumberOption MAXDEPTH = new NumberOption("maxDepth", Integer.MAX_VALUE);
-		public static final NumberOption MAXFILES = new NumberOption("maxFiles", Integer.MAX_VALUE);
+		public static final NumberOption MAX_DEPTH = new NumberOption("max-depth", Integer.MAX_VALUE);
+		public static final NumberOption MAX_FILES = new NumberOption("max-files", Integer.MAX_VALUE);
 		/** add attrs eg size. */
-		public static final BooleanOption SHOWFILEINFO = new BooleanOption("showFileInfo", false);
+		public static final BooleanOption INCLUDE_INFO = new BooleanOption("include-info", false);
+		public static final BooleanOption FOLLOW_LINKS = new BooleanOption("follow-links", false);
 		/** Query base-uri. */
 		public static final StringOption INCLUDE_FILTER = new StringOption("include-filter");
 		public static final StringOption EXCLUDE_FILTER = new StringOption("exclude-filter");
+	    public static final StringOption SKIP_FILTER = new StringOption("skip-filter");
 	}
 
 	public Value filewalk(final String path, final Map options) throws IOException, QueryException {
 		//FnTrace.trace("BY".getBytes(), "TEST".getBytes(), queryContext);
 		Path startingDir = Paths.get(path);
-		EnumSet<FileVisitOption> walkopts = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
 
-		int maxDepth=SimpleOptions.mapOption(options,"maxDepth",Integer.MAX_VALUE);
+		boolean followLinks=SimpleOptions.mapOption(options,"follow-links",false);
+		EnumSet<FileVisitOption> walkopts = (followLinks)?EnumSet.of(FileVisitOption.FOLLOW_LINKS):EnumSet.noneOf(FileVisitOption.class);
+
+		int maxDepth=SimpleOptions.mapOption(options,"max-depth",Integer.MAX_VALUE);
+		if(-1==maxDepth)maxDepth=Integer.MAX_VALUE;
 
 		// String s=options.dbl(ii)
 		Walker walk = new Walker(options, queryContext);

src/java/com/quodatum/file/SimpleOptions.java

@@ -8,6 +8,7 @@
 import org.basex.query.value.item.Str;
 import org.basex.query.value.map.Map;
 import org.basex.util.InputInfo;
+import org.basex.util.Token;
 
 public class SimpleOptions {
 
@@ -41,5 +42,19 @@ static  boolean mapOption(Map m, String skey, boolean def) throws QueryException
             return def;
         }
     }
-
+    // get int key from map or default def if missing/invalid 
+    static  String mapOption(Map m, String skey, String def) throws QueryException {
+        Item key = Str.get(skey);
+        InputInfo ii = new InputInfo("XQueryMaps.java", 0, 0);
+        Value v = m.get(key, ii);
+        if (v.isEmpty()) {
+            return def;
+        }
+        Item item = v.itemAt(0);
+        if (item.type.isStringOrUntyped()) {
+            return Token.string(item.string(null));
+        } else {
+            return def;
+        }
+    }
 }

src/java/com/quodatum/file/Walker.java

@@ -2,6 +2,10 @@
 
 /*
  * file walker
+ * @author andy bunce
+ * @copyright Quodatum Ltd
+ * @date 2015-2017
+ * @licence Apache 2
  * @see https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileVisitResult.html
  */
 import static java.nio.file.FileVisitResult.*;
@@ -11,40 +15,55 @@
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
-
 import org.basex.build.MemBuilder;
 import org.basex.build.SingleParser;
 import org.basex.core.MainOptions;
 import org.basex.io.IOContent;
 import org.basex.query.QueryContext;
 import org.basex.query.QueryException;
+import org.basex.query.func.fn.FnTrace;
 import org.basex.query.value.map.Map;
 import org.basex.query.value.node.ANode;
 import org.basex.query.value.node.DBNode;
 import org.basex.util.Atts;
 import org.basex.util.Token;
 
+@SuppressWarnings("unused")
 public class Walker extends SimpleFileVisitor<Path> {
-    
+
     final byte[] C_DIR = Token.token("c:directory");
     final byte[] C_FILE = Token.token("c:file");
     final byte[] C_ERR = Token.token("c:error");
     final byte[] NAME = Token.token("name");
     final byte[] BASE = Token.token("xml:base");
     final byte[] LAST_MODIFIED = Token.token("last-modified");
     final byte[] SIZE = Token.token("size");
+    final byte[] COUNT = Token.token("count");
     final Atts NSP = new Atts(Token.token("c"), Token.token("http://www.w3.org/ns/xproc-step"));
 
-    SingleParser singleParser=new SingleParser(new IOContent(""),MainOptions.get()){@Override protected void parse()throws IOException{}};
+    SingleParser singleParser = new SingleParser(new IOContent(""), MainOptions.get()) {
+        @Override
+        protected void parse() throws IOException {
+        }
+    };
     MemBuilder memBuilder = new MemBuilder("", singleParser);
-    private boolean showFileInfo=false;
+    private boolean showFileInfo = false;
     private int maxFiles;
     private int filesFound = 0;
-
+    private String includeFilter;
+    private String excludeFilter;
+    private String skipFilter;
+    private QueryContext queryContext;
+
     public Walker(Map options, QueryContext queryContext) throws IOException, QueryException {
         // options.get("showInfo", null);
-        showFileInfo=SimpleOptions.mapOption(options,"showFileInfo",false);
-        maxFiles=SimpleOptions.mapOption(options,"maxFiles",Integer.MAX_VALUE);
+        this.queryContext=queryContext;
+
+        showFileInfo = SimpleOptions.mapOption(options, "include-info", false);
+        maxFiles = SimpleOptions.mapOption(options, "max-files", Integer.MAX_VALUE);
+        includeFilter = SimpleOptions.mapOption(options, "include-filter", null);
+        excludeFilter = SimpleOptions.mapOption(options, "exclude-filter", null);
+        skipFilter = SimpleOptions.mapOption(options, "skip-filter", null);
         memBuilder.init();
     }
 
@@ -56,7 +75,18 @@ public ANode result() {
     // each type of file.
     @Override
     public FileVisitResult visitFile(Path file, BasicFileAttributes attr) throws IOException {
-        Atts atts = new Atts(NAME, Token.token(file.getFileName().toString()));
+        String name = file.getFileName().toString();
+        boolean use = true;
+        if (includeFilter != null) {
+            use = name.matches(includeFilter);
+        }
+        if (excludeFilter != null) {
+            use = use && !name.matches(excludeFilter);
+        }
+        if (!use)
+            return CONTINUE;
+
+        Atts atts = new Atts(NAME, Token.token(name));
         if (showFileInfo) {
             atts.add(LAST_MODIFIED, Token.token(attr.lastModifiedTime().toString()));
             atts.add(SIZE, Token.token(Long.toString(attr.size())));
@@ -65,26 +95,37 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attr) throws IOE
         if (attr.isDirectory()) {
             atts.add(BASE, Token.token(file.toUri().toString()));
             memBuilder.emptyElem(C_DIR, atts, NSP);
-        }else{
-            memBuilder.emptyElem(C_FILE, atts, NSP);
+        } else {
             filesFound++;
+            memBuilder.emptyElem(C_FILE, atts, NSP);
+
         }
-        return (filesFound<maxFiles)?CONTINUE:TERMINATE;
+        return (filesFound < maxFiles) ? CONTINUE : SKIP_SIBLINGS;
     }
 
     // Print each directory visited.
     @Override
     public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
-        Atts atts = new Atts();
-        atts.add(NAME, Token.token(dir.getFileName().toString()));
-        atts.add(BASE, Token.token(dir.toUri().toString()));
-        if (showFileInfo) {
-            atts.add(LAST_MODIFIED, Token.token(attrs.lastModifiedTime().toString()));
-            atts.add(SIZE, Token.token(Long.toString(attrs.size())));
+        String name = dir.getFileName().toString();
+        boolean skip=false;
+        if (skipFilter != null) {
+            skip = name.matches(skipFilter);
+        }
+        if (skip) return SKIP_SUBTREE;
+
+        if (filesFound < maxFiles) {
+            Atts atts = new Atts();
+            atts.add(NAME, Token.token(name));
+            atts.add(BASE, Token.token(dir.toUri().toString()));
+            if (showFileInfo) {
+                atts.add(LAST_MODIFIED, Token.token(attrs.lastModifiedTime().toString()));
+                atts.add(SIZE, Token.token(Long.toString(attrs.size())));
+            }
+            memBuilder.openElem(C_DIR, atts, NSP);
+            return CONTINUE;
+        } else {
+            return SKIP_SUBTREE;
         }
-        ;
-        memBuilder.openElem(C_DIR, atts, NSP);
-        return CONTINUE;
     }
 
     // Print each directory visited.
@@ -101,7 +142,8 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx
     // is thrown.
     @Override
     public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
-        Atts atts = new Atts(NAME, Token.token(file.getFileName().toString()));
+        String name = file.getFileName().toString();
+        Atts atts = new Atts(NAME, Token.token(name));
         memBuilder.emptyElem(C_ERR, atts, NSP);
         return CONTINUE;
     }

src/main/basex.xml

@@ -1,4 +1,4 @@
 <package xmlns="http://www.basex.org/modules/pkg">
-   <jar>quodatum-files-0.3.9.jar</jar>
+   <jar>quodatum-files-0.5.0.jar</jar>
    <class>com.quodatum.file.Runner</class>
 </package>

src/main/expath-pkg.xml

@@ -1,7 +1,7 @@
 <package xmlns="http://expath.org/ns/pkg"
          name="https://github.com/Quodatum/file-walker"
          abbrev="quodatum-files"
-         version="0.3.9"
+         version="0.5.0"
          spec="1.0">
 
    <title>File list and search in the XProc style</title>

src/test/smoke-test.xq

@@ -3,9 +3,13 @@ declare namespace c="http://www.w3.org/ns/xproc-step";
 declare variable $large:="Z:\pictures\Pictures";
 declare variable $small:="Z:\recordings\radio";
 declare variable $local:="C:\Users\andy\Desktop\radio";
-
-let $r:= fw:directory-list($large,map{"maxDepth":1,"showFileInfo":true()})
-return $r
+let $opts:=map{
+    "include-info":true(),
+    "max-files":6,
+    "exclude-filter":"dd" 
+}
+let $r:= fw:directory-list($small,$opts)
+return $r