From 7c9535efee200dd208abec75b3b652d1e88a585a Mon Sep 17 00:00:00 2001 From: ssarnobat Date: Sun, 6 Nov 2016 14:05:10 -0800 Subject: [PATCH] Minor changes (untested) --- server.groovy | 6 +-- video_download.groovy | 103 +++++++++++++++++++++++++++++++++++++ videos_get_undownloaded.sh | 5 ++ 3 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 video_download.groovy create mode 100644 videos_get_undownloaded.sh diff --git a/server.groovy b/server.groovy index b6c689e..c40ab1d 100644 --- a/server.groovy +++ b/server.groovy @@ -87,8 +87,8 @@ public class Yurl { // Gets stored here: http://192.168.1.2:28017/cache/items/ private static final boolean MONGODB_ENABLED = YurlResource.MongoDbCache.ENABLED; private static final String CHROMEDRIVER_PATH = "/home/sarnobat/github/yurl/chromedriver"; - private static final String YOUTUBE_DOWNLOAD = "/home/sarnobat/bin/youtube_download"; - private static final Integer ROOT_ID = 45; + public static final String YOUTUBE_DOWNLOAD = "/home/sarnobat/bin/youtube_download"; + public static final Integer ROOT_ID = 45; private static final String CYPHER_URI = "http://netgear.rohidekar.com:7474/db/data/cypher"; private static final String TARGET_DIR_PATH = "/media/sarnobat/Unsorted/Videos/"; private static final String QUEUE_FILE = "/home/sarnobat/sarnobat.git/"; @@ -920,7 +920,7 @@ public class Yurl { Process p = new ProcessBuilder() .directory(Paths.get(targetDirPath).toFile()) .command( - ImmutableList.of(YOUTUBE_DOWNLOAD, + ImmutableList.of(Yurl.YOUTUBE_DOWNLOAD, iVideoUrl)).inheritIO().start(); p.waitFor(); if (p.exitValue() == 0) { diff --git a/video_download.groovy b/video_download.groovy new file mode 100644 index 0000000..b00d738 --- /dev/null +++ b/video_download.groovy @@ -0,0 +1,103 @@ +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Map; + +import javax.ws.rs.core.MediaType; + +import org.apache.commons.io.IOUtils; +import org.json.JSONObject; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.api.json.JSONConfiguration; + +public class VideoDownloadNeo4j { + + private static final String YOUTUBE_DOWNLOAD = "/home/sarnobat/bin/youtube_download"; + private static final Integer ROOT_ID = 45; + private static final String CYPHER_URI = "http://netgear.rohidekar.com:7474/db/data/cypher"; + private static final String TARGET_DIR_PATH = "/media/sarnobat/Unsorted/Videos/"; + + public static void main(String[] args) { + String iVideoUrl = args[0]; + + try { + Process p = new ProcessBuilder() + .directory(Paths.get(TARGET_DIR_PATH).toFile()) + .command( + ImmutableList.of(YOUTUBE_DOWNLOAD, + iVideoUrl)).inheritIO().start(); + p.waitFor(); + if (p.exitValue() == 0) { + System.out + .println("YurlWorldResource.downloadVideoInSeparateThread() - successfully downloaded " + + iVideoUrl); + writeSuccessToDb(iVideoUrl); + } else { + System.err + .println("YurlWorldResource.downloadVideoInSeparateThread() - error downloading " + + iVideoUrl); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + private static void writeSuccessToDb(final String iVideoUrl) + throws IOException { + execute("start n=node(*) WHERE n.url = {url} SET n.downloaded_video = {date}", + ImmutableMap. of("url", iVideoUrl, + "date", new Long(System.currentTimeMillis())), true, "downloadVideo()"); + System.out.println("downloadVideo() - Download recorded in database"); + } + + private static JSONObject execute(String iCypherQuery, + Map iParams, boolean doLogging, String... iCommentPrefix) { + String commentPrefix = iCommentPrefix.length > 0 ? iCommentPrefix[0] + " " : ""; + if (doLogging) { + System.out.println(commentPrefix + " - \t" + iCypherQuery); + System.out.println(commentPrefix + "- \tparams - " + iParams); + } + ClientConfig clientConfig = new DefaultClientConfig(); + clientConfig.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, + Boolean.TRUE); + + // POST {} to the node entry point URI + ClientResponse theResponse = Client.create(clientConfig).resource( + CYPHER_URI) + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON).entity("{ }") + .post(ClientResponse.class, ImmutableMap + . of("query", iCypherQuery, "params", + Preconditions.checkNotNull(iParams))); + if (theResponse.getStatus() != 200) { + System.out.println(commentPrefix + "FAILED:\n\t" + iCypherQuery + "\n\tparams: " + + iParams); + try { + throw new RuntimeException(IOUtils.toString(theResponse.getEntityInputStream())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + String theNeo4jResponse ; + try { + // Do not inline this. We need to close the stream after + // copying + theNeo4jResponse = IOUtils.toString(theResponse.getEntityInputStream()); + theResponse.getEntityInputStream().close(); + theResponse.close(); + if (doLogging) { + System.out.println(commentPrefix + "end"); + } + return new JSONObject(theNeo4jResponse); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/videos_get_undownloaded.sh b/videos_get_undownloaded.sh new file mode 100644 index 0000000..f59c7b6 --- /dev/null +++ b/videos_get_undownloaded.sh @@ -0,0 +1,5 @@ +curl -X POST -H 'Content-type: application/json' \ + 'http://netgear.rohidekar.com:7474/db/data/cypher' -d ' + { + "query": "START n=node(37658) MATCH n-->c WHERE HAS(c.url) AND NOT HAS(c.downloaded_video) RETURN c.url", "params":{} + }'