From df26e2e0c3e36ed02f87e249bee788d54886e9dd Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Mon, 26 Dec 2016 21:49:44 +0100 Subject: [PATCH 1/8] libtracker-miner: Fully stop TrackerFileNotifier operations on errors It oddly tried to just keep going, which may result in spurious deletes or partial inserts. Sounds better to just drop the ball on the directory at hand, it should be correctly indexed eventually. --- src/libtracker-miner/tracker-file-notifier.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c index 4d3ec2467..5b1abe095 100644 --- a/src/libtracker-miner/tracker-file-notifier.c +++ b/src/libtracker-miner/tracker-file-notifier.c @@ -779,9 +779,11 @@ sparql_contents_query_cb (GObject *object, cursor = tracker_sparql_connection_query_finish (TRACKER_SPARQL_CONNECTION (object), result, &error); if (error) { - if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) - goto out; - g_warning ("Could not query directory contents: %s\n", error->message); + if (!g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) { + g_warning ("Could not query directory contents: %s\n", error->message); + finish_current_directory (notifier, TRUE); + } + goto out; } notifier = user_data; @@ -865,17 +867,19 @@ sparql_files_query_cb (GObject *object, GFile *directory; guint flags; + notifier = data->notifier; + priv = notifier->priv; + cursor = tracker_sparql_connection_query_finish (TRACKER_SPARQL_CONNECTION (object), result, &error); if (error) { - if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) - goto out; - g_warning ("Could not query indexed files: %s\n", error->message); + if (!g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) { + g_warning ("Could not query indexed files: %s\n", error->message); + finish_current_directory (notifier, TRUE); + } + goto out; } - notifier = data->notifier; - priv = notifier->priv; - if (cursor) { sparql_files_query_populate (notifier, cursor, TRUE); g_object_unref (cursor); -- 2.14.2 From 1f5ac7a8c3db62d23c0148429905eeaf5c45b36d Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Sun, 16 Apr 2017 11:58:00 -0300 Subject: [PATCH 2/8] libtracker-miner: Shuffle refcount handling when queueing back a file The file might or might not be inserted to the queue, which meant that the extra ref created outside the call might never dropped if the file didn't end up inserted again. Fix this by doing the refcount increase when actually inserting the file back in the queue. Reported by Jose M. Arroyo . --- src/libtracker-miner/tracker-miner-fs.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c index 02b4a4755..15ac82f4d 100644 --- a/src/libtracker-miner/tracker-miner-fs.c +++ b/src/libtracker-miner/tracker-miner-fs.c @@ -2109,11 +2109,10 @@ should_wait (TrackerMinerFS *fs, } static gboolean -item_reenqueue_full (TrackerMinerFS *fs, - TrackerPriorityQueue *item_queue, - GFile *queue_file, - gpointer queue_data, - gint priority) +item_enqueue_again (TrackerMinerFS *fs, + TrackerPriorityQueue *item_queue, + GFile *queue_file, + gint priority) { gint reentry_counter; gchar *uri; @@ -2126,7 +2125,7 @@ item_reenqueue_full (TrackerMinerFS *fs, g_object_set_qdata (G_OBJECT (queue_file), fs->priv->quark_reentry_counter, GINT_TO_POINTER (reentry_counter + 1)); - tracker_priority_queue_add (item_queue, queue_data, priority); + tracker_priority_queue_add (item_queue, g_object_ref (queue_file), priority); should_wait = TRUE; } else { @@ -2147,15 +2146,6 @@ item_reenqueue_full (TrackerMinerFS *fs, return should_wait; } -static gboolean -item_reenqueue (TrackerMinerFS *fs, - TrackerPriorityQueue *item_queue, - GFile *queue_file, - gint priority) -{ - return item_reenqueue_full (fs, item_queue, queue_file, queue_file, priority); -} - static QueueState item_queue_get_next_file (TrackerMinerFS *fs, GFile **file, @@ -2601,8 +2591,8 @@ item_queue_handlers_cb (gpointer user_data) * ensured, tasks are inserted at a higher priority so they * are processed promptly anyway. */ - item_reenqueue (fs, item_queue, g_object_ref (parent), priority - 1); - item_reenqueue (fs, item_queue, g_object_ref (file), priority); + item_enqueue_again (fs, item_queue, parent, priority - 1); + item_enqueue_again (fs, item_queue, file, priority); keep_processing = TRUE; } -- 2.14.2 From cc917a0c029ebc46a1fbca5d73a9bd67b27e2e57 Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Thu, 6 Jul 2017 11:19:17 +0200 Subject: [PATCH 3/8] libtracker-miner: Fix warnings if move ops happened during initial crawling The checks to notify about indexing having finished on TrackerIndexingTree roots were mistaking ItemMovedData* with GFile*, which lead to warnings. This should be harmless, the signal might be possibly emitted before the move op is dispatched, that's all. --- src/libtracker-miner/tracker-miner-fs.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c index 15ac82f4d..baa2c9a7a 100644 --- a/src/libtracker-miner/tracker-miner-fs.c +++ b/src/libtracker-miner/tracker-miner-fs.c @@ -1110,6 +1110,15 @@ miner_resumed (TrackerMiner *miner) } } +static gboolean +item_moved_data_has_prefix (gpointer data, + gpointer user_data) +{ + ItemMovedData *moved_item = data; + GFile *prefix = user_data; + + return g_file_has_prefix (moved_item->file, prefix); +} static void miner_ignore_next_update (TrackerMiner *miner, const GStrv urls) @@ -1165,7 +1174,7 @@ notify_roots_finished (TrackerMinerFS *fs, (tracker_priority_queue_find (fs->priv->items_created, NULL, (GEqualFunc) g_file_has_prefix, root) || tracker_priority_queue_find (fs->priv->items_updated, NULL, (GEqualFunc) g_file_has_prefix, root) || tracker_priority_queue_find (fs->priv->items_deleted, NULL, (GEqualFunc) g_file_has_prefix, root) || - tracker_priority_queue_find (fs->priv->items_moved, NULL, (GEqualFunc) g_file_has_prefix, root) || + tracker_priority_queue_find (fs->priv->items_moved, NULL, (GEqualFunc) item_moved_data_has_prefix, root) || tracker_priority_queue_find (fs->priv->items_writeback, NULL, (GEqualFunc) g_file_has_prefix, root))) { continue; } -- 2.14.2 From 74ad92e34d8f58e9c8f015bf9f3173e3f738a23c Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Sun, 9 Jul 2017 19:08:54 +0200 Subject: [PATCH 4/8] libtracker-miner: Ensure sparql buffer keeps flushing The sparql buffer might get full again with new tasks before the update_array operation for the current batch returned. In this case nothing will kick the TrackerMinerFS again, nor the SPARQL buffer from flushing again. Fix this by just flushing again, the miner will follow as soon as the SPARQL buffer is below limits. --- src/libtracker-miner/tracker-sparql-buffer.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libtracker-miner/tracker-sparql-buffer.c b/src/libtracker-miner/tracker-sparql-buffer.c index 5d0be93a4..95f9c8ede 100644 --- a/src/libtracker-miner/tracker-sparql-buffer.c +++ b/src/libtracker-miner/tracker-sparql-buffer.c @@ -246,6 +246,7 @@ tracker_sparql_buffer_update_array_cb (GObject *object, gpointer user_data) { TrackerSparqlBufferPrivate *priv; + TrackerSparqlBuffer *buffer; GError *global_error = NULL; GPtrArray *sparql_array_errors; UpdateArrayData *update_data; @@ -253,7 +254,8 @@ tracker_sparql_buffer_update_array_cb (GObject *object, /* Get arrays of errors and queries */ update_data = user_data; - priv = TRACKER_SPARQL_BUFFER (update_data->buffer)->priv; + buffer = TRACKER_SPARQL_BUFFER (update_data->buffer); + priv = buffer->priv; priv->n_updates--; g_debug ("(Sparql buffer) Finished array-update with %u tasks", @@ -338,6 +340,10 @@ tracker_sparql_buffer_update_array_cb (GObject *object, if (global_error) { g_error_free (global_error); } + + if (tracker_task_pool_limit_reached (TRACKER_TASK_POOL (buffer))) { + tracker_sparql_buffer_flush (buffer, "SPARQL buffer limit reached (after flush)"); + } } static gchar * -- 2.14.2 From 534a1839a0f6e5bb71af7bc51b79f9d9e215266b Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Mon, 25 Sep 2017 12:26:33 +0200 Subject: [PATCH 5/8] libtracker-miner: Check directory updates before dropping mtime info file_notifier_traverse_tree() has the side effect of deleting mtime data, as it's presumably not needed anymore, except here. Check whether the directory was updated before notifying of the files inside, so we can thoroughly check for deleted content. https://bugzilla.gnome.org/show_bug.cgi?id=786132 --- src/libtracker-miner/tracker-file-notifier.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c index 5b1abe095..433f79524 100644 --- a/src/libtracker-miner/tracker-file-notifier.c +++ b/src/libtracker-miner/tracker-file-notifier.c @@ -863,6 +863,7 @@ sparql_files_query_cb (GObject *object, TrackerFileNotifierPrivate *priv; TrackerFileNotifier *notifier; TrackerSparqlCursor *cursor; + gboolean directory_modified; GError *error = NULL; GFile *directory; guint flags; @@ -885,13 +886,15 @@ sparql_files_query_cb (GObject *object, g_object_unref (cursor); } - file_notifier_traverse_tree (notifier, data->max_depth); directory = priv->current_index_root->current_dir; flags = priv->current_index_root->flags; + directory_modified = file_notifier_is_directory_modified (notifier, directory); + + file_notifier_traverse_tree (notifier, data->max_depth); if ((flags & TRACKER_DIRECTORY_FLAG_CHECK_DELETED) != 0 || priv->current_index_root->current_dir_content_filtered || - file_notifier_is_directory_modified (notifier, directory)) { + directory_modified) { /* The directory has updated its mtime, this means something * was either added or removed in the mean time. Crawling * will always find all newly added files. But still, we -- 2.14.2 From c48c677e3a5ba0045868049e28a52c5e5e49b980 Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Mon, 25 Sep 2017 12:12:47 +0200 Subject: [PATCH 6/8] libtracker-miner: Always fallback to URN query when queueing file Tracker may end up with nfo:FileDataObject prior to handling monitor events. Be it leftover data from previous bugs, explicit "tracker index" calls, or data from some other application. As we can't be really sure of the data consistence, always fallback to a URN query so we don't break nie:url UNIQUE constraint (inverse functional property in SPARQL parlance). https://bugzilla.gnome.org/show_bug.cgi?id=786132 --- src/libtracker-miner/tracker-miner-fs.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c index baa2c9a7a..1a4af1c12 100644 --- a/src/libtracker-miner/tracker-miner-fs.c +++ b/src/libtracker-miner/tracker-miner-fs.c @@ -2821,12 +2821,11 @@ miner_fs_cache_file_urn (TrackerMinerFS *fs, static void miner_fs_queue_file (TrackerMinerFS *fs, TrackerPriorityQueue *item_queue, - GFile *file, - gboolean query_urn) + GFile *file) { gint priority; - miner_fs_cache_file_urn (fs, file, query_urn); + miner_fs_cache_file_urn (fs, file, TRUE); priority = miner_fs_get_queue_priority (fs, file); tracker_priority_queue_add (item_queue, g_object_ref (file), priority); } @@ -2981,7 +2980,7 @@ check_item_queues (TrackerMinerFS *fs, */ g_debug (" Found matching unhandled CREATED event " "for source file, merging both events together"); - miner_fs_queue_file (fs, fs->priv->items_created, other_file, FALSE); + miner_fs_queue_file (fs, fs->priv->items_created, other_file); return FALSE; } @@ -3016,7 +3015,7 @@ file_notifier_file_created (TrackerFileNotifier *notifier, TrackerMinerFS *fs = user_data; if (check_item_queues (fs, QUEUE_CREATED, file, NULL)) { - miner_fs_queue_file (fs, fs->priv->items_created, file, FALSE); + miner_fs_queue_file (fs, fs->priv->items_created, file); item_queue_handlers_set_up (fs); } } @@ -3039,7 +3038,7 @@ file_notifier_file_deleted (TrackerFileNotifier *notifier, } if (check_item_queues (fs, QUEUE_DELETED, file, NULL)) { - miner_fs_queue_file (fs, fs->priv->items_deleted, file, FALSE); + miner_fs_queue_file (fs, fs->priv->items_deleted, file); item_queue_handlers_set_up (fs); } } @@ -3070,7 +3069,7 @@ file_notifier_file_updated (TrackerFileNotifier *notifier, GINT_TO_POINTER (TRUE)); } - miner_fs_queue_file (fs, fs->priv->items_updated, file, TRUE); + miner_fs_queue_file (fs, fs->priv->items_updated, file); item_queue_handlers_set_up (fs); } } @@ -3416,7 +3415,7 @@ tracker_miner_fs_directory_remove_full (TrackerMinerFS *fs, * to preserve remove_full() semantics. */ trace_eq_push_tail ("DELETED", file, "on remove full"); - miner_fs_queue_file (fs, fs->priv->items_deleted, file, FALSE); + miner_fs_queue_file (fs, fs->priv->items_deleted, file); item_queue_handlers_set_up (fs); } @@ -3460,7 +3459,7 @@ check_file_parents (TrackerMinerFS *fs, for (p = parents; p; p = p->next) { trace_eq_push_tail ("UPDATED", p->data, "checking file parents"); - miner_fs_queue_file (fs, fs->priv->items_updated, p->data, TRUE); + miner_fs_queue_file (fs, fs->priv->items_updated, p->data); g_object_unref (p->data); } -- 2.14.2 From 9df5d3cae1c79bdb3c5f930bf665fe5d1f442547 Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Sun, 24 Sep 2017 12:04:22 +0200 Subject: [PATCH 7/8] tracker-miner-fs: Explicitly ignore non-native mounts tracker-miner-fs does rely in a few places to index local content exclusively. Make it explicitly ignore non-native mounts, they won't be indexed anyway, but we can spare trying to create the tracker:Volume and mount root's nfo:FileDataObject for these. Fixes nie:url constraint errors and other warnings when trying to deal with those. https://bugzilla.gnome.org/show_bug.cgi?id=786132 --- src/miners/fs/tracker-storage.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/miners/fs/tracker-storage.c b/src/miners/fs/tracker-storage.c index 9ec59dbfd..6114a6fd6 100644 --- a/src/miners/fs/tracker-storage.c +++ b/src/miners/fs/tracker-storage.c @@ -550,8 +550,16 @@ mount_add (TrackerStorage *storage, /* Get root path of the mount */ root = g_mount_get_root (mount); - mount_path = g_file_get_path (root); + if (!g_file_is_native (root)) { + gchar *uri = g_file_get_uri (root); + g_debug ("Ignoring mount '%s', URI '%s' is not native", + mount_name, uri); + g_free (uri); + return; + } + + mount_path = g_file_get_path (root); g_debug ("Found '%s' mounted on path '%s'", mount_name, mount_path); -- 2.14.2 From 907a867cd72827d5b24d6d8474d9e76d6ff095aa Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Sun, 10 Sep 2017 17:01:26 +0200 Subject: [PATCH 8/8] applications: Ignore broken symlinks We still do need to query mtime from the symlink itself in order to keep filesystem accounting happy, so wait for ENOENT while doing g_key_file_load_from_file() in order to figure out we are dealing with a broken link (or a no longer existing file, which is also possible). This avoids spewing any warnings with those. https://bugzilla.gnome.org/show_bug.cgi?id=786132 --- src/miners/apps/tracker-miner-applications.c | 34 ++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/miners/apps/tracker-miner-applications.c b/src/miners/apps/tracker-miner-applications.c index 749d3e4d2..a2d1f78d8 100644 --- a/src/miners/apps/tracker-miner-applications.c +++ b/src/miners/apps/tracker-miner-applications.c @@ -430,8 +430,7 @@ get_desktop_key_file (GFile *file, key_file = g_key_file_new (); *type = NULL; - if (!g_key_file_load_from_file (key_file, path, G_KEY_FILE_NONE, NULL)) { - g_set_error (error, miner_applications_error_quark, 0, "Couldn't load desktop file:'%s'", path); + if (!g_key_file_load_from_file (key_file, path, G_KEY_FILE_NONE, error)) { g_key_file_free (key_file); g_free (path); return NULL; @@ -943,6 +942,7 @@ process_file_cb (GObject *object, ProcessApplicationData *data; GFileInfo *file_info; GError *error = NULL; + GFileType file_type; GFile *file; data = user_data; @@ -956,21 +956,27 @@ process_file_cb (GObject *object, return; } - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) { + file_type = g_file_info_get_file_type (file_info); + + if (file_type == G_FILE_TYPE_DIRECTORY) { process_directory (data, file_info, &error); - } else { + } else if (file_type == G_FILE_TYPE_REGULAR || + file_type == G_FILE_TYPE_SYMBOLIC_LINK) { data->key_file = get_desktop_key_file (file, &data->type, &error); if (!data->key_file) { - gchar *uri; - - uri = g_file_get_uri (file); - g_warning ("Couldn't properly parse desktop file '%s': '%s'", - uri, - error ? error->message : "unknown error"); - g_free (uri); - g_clear_error (&error); - - error = g_error_new_literal (miner_applications_error_quark, 0, "File is not a key file"); + /* Ignore broken symlinks */ + if (!g_error_matches (error, G_FILE_ERROR, G_FILE_ERROR_NOENT)) { + gchar *uri; + + uri = g_file_get_uri (file); + g_warning ("Couldn't properly parse desktop file '%s': '%s'", + uri, + error ? error->message : "unknown error"); + g_free (uri); + g_clear_error (&error); + + error = g_error_new_literal (miner_applications_error_quark, 0, "File is not a key file"); + } } else if (g_key_file_get_boolean (data->key_file, GROUP_DESKTOP_ENTRY, "Hidden", NULL)) { error = g_error_new_literal (miner_applications_error_quark, 0, "Desktop file is 'hidden', not gathering metadata for it"); } else { -- 2.14.2