From fc4e314dbc2937e13e0d8e1fb262fae15fa074cd Mon Sep 17 00:00:00 2001 From: whatdoineed2do/Ray Date: Sun, 24 Jan 2021 13:35:50 +0000 Subject: [PATCH 1/6] [db] *_ping_excl_bymatch() for partial scans --- src/db.c | 36 ++++++++++++++++++++++++++++++++++++ src/db.h | 9 +++++++++ 2 files changed, 45 insertions(+) diff --git a/src/db.c b/src/db.c index 8b23af16a2..9a6255bded 100644 --- a/src/db.c +++ b/src/db.c @@ -3056,6 +3056,18 @@ db_file_ping_bymatch(const char *path, int isdir) #undef Q_TMPL_NODIR } +void +db_file_ping_excl_bymatch(const char *path) +{ +#define Q_TMPL_DIR "UPDATE files SET db_timestamp = %" PRIi64 " WHERE path NOT LIKE '%q/%%';" + char *query; + + query = sqlite3_mprintf(Q_TMPL_DIR, (int64_t)time(NULL), path); + db_query_run(query, 1, 0); + +#undef Q_TMPL_DIR +} + char * db_file_path_byid(int id) { @@ -3639,6 +3651,18 @@ db_pl_ping_bymatch(const char *path, int isdir) #undef Q_TMPL_NODIR } +void +db_pl_ping_excl_bymatch(const char *path) +{ +#define Q_TMPL_DIR "UPDATE playlists SET db_timestamp = %" PRIi64 " WHERE path NOT LIKE '%q/%%';" + char *query; + + query = sqlite3_mprintf(Q_TMPL_DIR, (int64_t)time(NULL), path); + db_query_run(query, 1, 0); + +#undef Q_TMPL_DIR +} + void db_pl_ping_items_bymatch(const char *path, int id) { @@ -4453,6 +4477,18 @@ db_directory_ping_bymatch(char *virtual_path) #undef Q_TMPL_DIR } +void +db_directory_ping_excl_bymatch(const char *virtual_path) +{ +#define Q_TMPL_DIR "UPDATE directories SET db_timestamp = %" PRIi64 " WHERE virtual_path <> '%q' OR virtual_path NOT LIKE '%q/%%';" + char *query; + + query = sqlite3_mprintf(Q_TMPL_DIR, (int64_t)time(NULL), virtual_path, virtual_path); + + db_query_run(query, 1, 0); +#undef Q_TMPL_DIR +} + void db_directory_disable_bymatch(const char *path, enum strip_type strip, uint32_t cookie) { diff --git a/src/db.h b/src/db.h index 7b3cf6317f..d4b9c765aa 100644 --- a/src/db.h +++ b/src/db.h @@ -682,6 +682,9 @@ db_file_ping_bypath(const char *path, time_t mtime_max); void db_file_ping_bymatch(const char *path, int isdir); +void +db_file_ping_excl_bymatch(const char *path); + char * db_file_path_byid(int id); @@ -746,6 +749,9 @@ db_pl_ping(int id); void db_pl_ping_bymatch(const char *path, int isdir); +void +db_pl_ping_excl_bymatch(const char *path); + void db_pl_ping_items_bymatch(const char *path, int id); @@ -827,6 +833,9 @@ db_directory_update(struct directory_info *di); void db_directory_ping_bymatch(char *virtual_path); +void +db_directory_ping_excl_bymatch(const char *virtual_path); + void db_directory_disable_bymatch(const char *path, enum strip_type strip, uint32_t cookie); From 3fbef5360fa9656a934075671ff937e0763d0656 Mon Sep 17 00:00:00 2001 From: whatdoineed2do/Ray Date: Sat, 23 Jan 2021 15:17:11 +0000 Subject: [PATCH 2/6] [library] Add rescan_path to library interface --- src/library.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/library.h | 8 +++++ 2 files changed, 96 insertions(+) diff --git a/src/library.c b/src/library.c index 4d4f842856..7d3a20c2b5 100644 --- a/src/library.c +++ b/src/library.c @@ -367,6 +367,81 @@ rescan(void *arg, int *ret) return COMMAND_END; } +static enum command_state +rescan_path(void *arg, int *ret) +{ + time_t starttime; + time_t endtime; + int i; + char *path = (char *)arg; + char virtual_path[PATH_MAX]; + struct stat st; + int ret1; + char *ptr; + + // drop any trailing '/' on path + ptr = path + strlen(path)-1; + while (ptr > path) + { + if (*ptr != '/') + break; + + *ptr = '\0'; + --ptr; + } + + listener_notify(LISTENER_UPDATE); + starttime = time(NULL); + + DPRINTF(E_LOG, L_LIB, "Library partial rescan triggered: '%s'\n", path); + ret1 = lstat(path, &st); + if (ret1 < 0 || (ret1 == 0 && (st.st_mode & S_IFMT) != S_IFDIR)) + { + DPRINTF(E_LOG, L_LIB, "Partial rescan on '%s' is not a valid directory\n", path); + goto out; + } + + // protecting everything else other than the request path + db_file_ping_excl_bymatch(path); + db_pl_ping_excl_bymatch(path); + ret1 = snprintf(virtual_path, sizeof(virtual_path), "/file:%s", path); + if ((ret1 < 0) || (ret1 >= sizeof(virtual_path))) + DPRINTF(E_LOG, L_SCAN, "Virtual path exceeds PATH_MAX (/file:%s)\n", path); + else + db_directory_ping_excl_bymatch(virtual_path); + + for (i = 0; sources[i]; i++) + { + if (!sources[i]->disabled && sources[i]->rescan_path && sources[i]->scan_kind == SCAN_KIND_FILES) + { + DPRINTF(E_INFO, L_LIB, "Rescan partial library source '%s'\n", db_scan_kind_label(sources[i]->scan_kind)); + sources[i]->rescan_path(path); + } + else + { + DPRINTF(E_INFO, L_LIB, "Library partial source '%s' is disabled\n", db_scan_kind_label(sources[i]->scan_kind)); + } + } + + purge_cruft(starttime, SCAN_KIND_FILES); + + DPRINTF(E_DBG, L_LIB, "Running post library partial scan jobs\n"); + db_hook_post_scan(); + +out: + endtime = time(NULL); + DPRINTF(E_LOG, L_LIB, "Library partial rescan completed in %.f sec (%d changes)\n", difftime(endtime, starttime), deferred_update_notifications); + scanning = false; + + if (handle_deferred_update_notifications()) + listener_notify(LISTENER_UPDATE | LISTENER_DATABASE); + else + listener_notify(LISTENER_UPDATE); + + *ret = 0; + return COMMAND_END; +} + static enum command_state metarescan(void *arg, int *ret) { @@ -760,6 +835,19 @@ library_rescan(enum scan_kind scan_kind) commands_exec_async(cmdbase, rescan, param); } +void +library_rescan_path(const char *path) +{ + if (scanning) + { + DPRINTF(E_INFO, L_LIB, "Scan already running, ignoring request to trigger a new init path scan\n"); + return; + } + + scanning = true; // TODO Guard "scanning" with a mutex + commands_exec_async(cmdbase, rescan_path, (void*)strdup(path)); +} + void library_metarescan(enum scan_kind scan_kind) { diff --git a/src/library.h b/src/library.h index 5201d951c7..247bf57859 100644 --- a/src/library.h +++ b/src/library.h @@ -100,6 +100,11 @@ struct library_source */ int (*write_metadata)(struct media_file_info *mfi); + /* + * Run rescan (called from the library thread) + */ + int (*rescan_path)(const char *path); + /* * Add an item to the library */ @@ -183,6 +188,9 @@ library_rescan(enum scan_kind library_source); /* * Same as library_rescan but also updates unmodified tracks and playlists */ +void +library_rescan_path(const char *path); + void library_metarescan(enum scan_kind library_source); From 42af092df5c4800779e6e2afddda3899bffdd1c3 Mon Sep 17 00:00:00 2001 From: whatdoineed2do/Ray Date: Sat, 23 Jan 2021 15:18:01 +0000 Subject: [PATCH 3/6] [scan] impl rescan_path for filescanner to allow directory specific scanning --- src/library/filescanner.c | 44 ++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/library/filescanner.c b/src/library/filescanner.c index 2bb208b462..eda9197ed7 100644 --- a/src/library/filescanner.c +++ b/src/library/filescanner.c @@ -151,7 +151,7 @@ static uint32_t incomingfiles_buffer[INCOMINGFILES_BUFFER_SIZE]; /* Forward */ static void -bulk_scan(int flags); +bulk_scan(const char *req_path, int flags); static int inofd_event_set(void); static void @@ -1003,7 +1003,7 @@ process_directories(char *root, int parent_id, int flags) /* Thread: scan */ static void -bulk_scan(int flags) +bulk_scan(const char *req_path, int flags) { cfg_t *lib; int ndirs; @@ -1029,6 +1029,18 @@ bulk_scan(int flags) { path = cfg_getnstr(lib, "directories", i); + // make sure path is in library if we've been asked to scan a specific path + if (req_path) + { + if (strncmp(path, req_path, strlen(path)) == 0) + path = (char*)req_path; + else + { + DPRINTF(E_LOG, L_SCAN, "Skipping request path: '%s', not in library directories\n", req_path); + continue; + } + } + parent_id = process_parent_directories(path); deref = realpath(path, NULL); @@ -1664,9 +1676,9 @@ filescanner_initscan() } if (cfg_getbool(cfg_getsec(cfg, "library"), "filescan_disable")) - bulk_scan(F_SCAN_BULK | F_SCAN_FAST); + bulk_scan(NULL, F_SCAN_BULK | F_SCAN_FAST); else - bulk_scan(F_SCAN_BULK); + bulk_scan(NULL, F_SCAN_BULK); if (!library_is_exiting()) { @@ -1684,7 +1696,24 @@ filescanner_rescan() inofd_event_unset(); // Clears all inotify watches db_watch_clear(); inofd_event_set(); - bulk_scan(F_SCAN_BULK | F_SCAN_RESCAN); + bulk_scan(NULL, F_SCAN_BULK | F_SCAN_RESCAN); + + if (!library_is_exiting()) + { + /* Enable inotify */ + event_add(inoev, NULL); + } + return 0; +} + +static int +filescanner_rescan_path(const char *path) +{ + DPRINTF(E_LOG, L_SCAN, "rescan triggered for '%s'\n", path); + + db_watch_delete_bypath((char*)path); + db_watch_delete_bymatch((char*)path); + bulk_scan(path, F_SCAN_BULK | F_SCAN_RESCAN); if (!library_is_exiting()) { @@ -1702,7 +1731,7 @@ filescanner_metarescan() inofd_event_unset(); // Clears all inotify watches db_watch_clear(); inofd_event_set(); - bulk_scan(F_SCAN_BULK | F_SCAN_METARESCAN); + bulk_scan(NULL, F_SCAN_BULK | F_SCAN_METARESCAN); if (!library_is_exiting()) { @@ -1719,7 +1748,7 @@ filescanner_fullrescan() inofd_event_unset(); // Clears all inotify watches inofd_event_set(); - bulk_scan(F_SCAN_BULK); + bulk_scan(NULL, F_SCAN_BULK); if (!library_is_exiting()) { @@ -2231,6 +2260,7 @@ struct library_source filescanner = .deinit = filescanner_deinit, .initscan = filescanner_initscan, .rescan = filescanner_rescan, + .rescan_path = filescanner_rescan_path, .metarescan = filescanner_metarescan, .fullrescan = filescanner_fullrescan, .write_metadata = filescanner_write_metadata, From f76003ca9ef549bdd61eaed32b17d9216d7966fe Mon Sep 17 00:00:00 2001 From: whatdoineed2do/Ray Date: Sat, 23 Jan 2021 15:19:27 +0000 Subject: [PATCH 4/6] [jsonapi] add param to 'api/update?path=....' endpoint to allow client request for path specific scans --- src/httpd_jsonapi.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/httpd_jsonapi.c b/src/httpd_jsonapi.c index 09984541d8..0d0df6ec9b 100644 --- a/src/httpd_jsonapi.c +++ b/src/httpd_jsonapi.c @@ -1230,10 +1230,16 @@ static int jsonapi_reply_update(struct httpd_request *hreq) { const char *param; + const char *param_path; param = httpd_query_value_find(hreq->query, "scan_kind"); + param_path = httpd_query_value_find(hreq->query, "path"); + + if (param_path) + library_rescan_path(param_path); + else + library_rescan(db_scan_kind_enum(param)); - library_rescan(db_scan_kind_enum(param)); return HTTP_NOCONTENT; } From de3268eddf6c56e44bca6be528563c3c34d198d2 Mon Sep 17 00:00:00 2001 From: whatdoineed2do/Ray Date: Fri, 29 Jan 2021 22:07:42 +0000 Subject: [PATCH 5/6] [db] add db_watch_enum_fetch() to return all watch_info --- src/db.c | 38 ++++++++++++++++++++++++++++++++++++-- src/db.h | 5 ++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/db.c b/src/db.c index 9a6255bded..40b60b6731 100644 --- a/src/db.c +++ b/src/db.c @@ -6530,8 +6530,8 @@ db_watch_cookie_known(uint32_t cookie) int db_watch_enum_start(struct watch_enum *we) { -#define Q_MATCH_TMPL "SELECT wd FROM inotify WHERE path LIKE '%q/%%';" -#define Q_COOKIE_TMPL "SELECT wd FROM inotify WHERE cookie = %" PRIi64 ";" +#define Q_MATCH_TMPL "SELECT wd,path FROM inotify WHERE path LIKE '%q/%%';" +#define Q_COOKIE_TMPL "SELECT wd,path FROM inotify WHERE cookie = %" PRIi64 ";" sqlite3_stmt *stmt; char *query; int ret; @@ -6616,6 +6616,40 @@ db_watch_enum_fetchwd(struct watch_enum *we, uint32_t *wd) return 0; } +int +db_watch_enum_fetch(struct watch_enum *we, struct watch_info *wi) +{ + int ret; + + wi->wd = 0; + wi->cookie = 0; + + if (!we->stmt) + { + DPRINTF(E_LOG, L_DB, "Watch enum not started!\n"); + return -1; + } + + ret = db_blocking_step(we->stmt); + if (ret == SQLITE_DONE) + { + DPRINTF(E_INFO, L_DB, "End of watch enum results\n"); + return 0; + } + else if (ret != SQLITE_ROW) + { + DPRINTF(E_LOG, L_DB, "Could not step: %s\n", sqlite3_errmsg(hdl)); + return -1; + } + + wi->wd = (uint32_t)sqlite3_column_int(we->stmt, 0); + if (wi->path) + snprintf(wi->path, PATH_MAX, (char*)sqlite3_column_text(we->stmt, 1)); + + return 0; +} + + #ifdef DB_PROFILE static int diff --git a/src/db.h b/src/db.h index d4b9c765aa..dce43dcc5b 100644 --- a/src/db.h +++ b/src/db.h @@ -1026,7 +1026,10 @@ int db_watch_enum_fetchwd(struct watch_enum *we, uint32_t *wd); int -db_backup(void); +db_watch_enum_fetch(struct watch_enum *we, struct watch_info *wi); + +int +db_backup(); int db_perthread_init(void); From 9999fbff7db3ba3cbc9065637b08bca3c91cf20c Mon Sep 17 00:00:00 2001 From: whatdoineed2do/Ray Date: Fri, 29 Jan 2021 22:09:16 +0000 Subject: [PATCH 6/6] [scan] partial scan ensure's watchers removed/readded for scanned dir --- src/library/filescanner.c | 60 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/library/filescanner.c b/src/library/filescanner.c index eda9197ed7..0916b2f7d9 100644 --- a/src/library/filescanner.c +++ b/src/library/filescanner.c @@ -1126,6 +1126,59 @@ watches_clear(uint32_t wd, char *path) return 0; } +static int +watches_clear_bypath(char *path) +{ + struct watch_info wi; + struct watch_enum we; + int ret; + + memset(&wi, 0, sizeof(struct watch_info)); + + wi.path = path; + db_watch_get_bypath(&wi, path); + watches_clear(wi.wd, path); + free(wi.path); + + memset(&we, 0, sizeof(struct watch_enum)); + we.match = ""; // get everything + + ret = db_watch_enum_start(&we); + if (ret < 0) + return -1; + + memset(&wi, 0, sizeof(struct watch_info)); + wi.path = malloc(PATH_MAX); + while (db_watch_enum_fetch(&we, &wi) == 0 && wi.wd) + { + inotify_rm_watch(inofd, wi.wd); + db_watch_delete_bypath(wi.path); + +#ifdef __linux__ + wi.wd = inotify_add_watch(inofd, wi.path, IN_ATTRIB | IN_CREATE | IN_DELETE | IN_CLOSE_WRITE | IN_MOVE | IN_DELETE | IN_MOVE_SELF); +#else + wi.wd = inotify_add_watch(inofd, wi.path, IN_CREATE | IN_DELETE | IN_MOVE); +#endif + if (wi.wd < 0) + { + DPRINTF(E_LOG, L_SCAN, "Failed to obtain watch: '%s' - %s\n", wi.path, strerror(errno)); + } + else + { + wi.cookie = 0; + db_watch_add(&wi); + } + + wi.wd = 0; + wi.path[0] = '\0'; + } + + db_watch_enum_end(&we); + + free(wi.path); + return 0; +} + /* Thread: scan */ static void process_inotify_dir(struct watch_info *wi, char *path, struct inotify_event *ie) @@ -1711,8 +1764,11 @@ filescanner_rescan_path(const char *path) { DPRINTF(E_LOG, L_SCAN, "rescan triggered for '%s'\n", path); - db_watch_delete_bypath((char*)path); - db_watch_delete_bymatch((char*)path); + inofd_event_unset(); // Clears all inotify watches, closing hdl + inofd_event_set(); // and get a new inotify hdl + + // readd watchers but exclude path and let the bulk_scan to readd + watches_clear_bypath((char*)path); bulk_scan(path, F_SCAN_BULK | F_SCAN_RESCAN); if (!library_is_exiting())