From 0a69894320cab7ef58944a382fa5d7d7c31c40d7 Mon Sep 17 00:00:00 2001 From: Chuddah Date: Sun, 23 Feb 2020 18:00:38 +0000 Subject: [PATCH 1/3] Attempt to reduce the number of calls to execute and fetchone. Profiling has shown that there are many calls to sqlite3.execute and fetchone which takes a signicant amount of time. A simple way of reducing these is to swap the 2-stage init of table row data into a unified add. Applying this to add_set and add_country yielded these results: Before changes ``` 281784 7.054 0.000 7.054 0.000 {method 'execute' of 'sqlite3.Cursor' objects} 127443 1.114 0.000 1.114 0.000 {method 'fetchone' of 'sqlite3.Cursor' objects} ``` After changes ``` 281714 7.492 0.000 7.492 0.000 {method 'execute' of 'sqlite3.Cursor' objects} 127373 1.217 0.000 1.217 0.000 {method 'fetchone' of 'sqlite3.Cursor' objects} ``` Note: The total time of fetchone has actually increased. I am hoping this was an abnormality on my machine and the actual reduction in the number of calls will permantly reduce this total time. --- jellyfin_kodi/objects/kodi/movies.py | 24 ++++-------------------- jellyfin_kodi/objects/kodi/queries.py | 16 ++++------------ 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/jellyfin_kodi/objects/kodi/movies.py b/jellyfin_kodi/objects/kodi/movies.py index 40c442a1..0515f2e0 100644 --- a/jellyfin_kodi/objects/kodi/movies.py +++ b/jellyfin_kodi/objects/kodi/movies.py @@ -37,16 +37,6 @@ class Movies(Kodi): return self.cursor.fetchone()[0] + 1 - def create_entry_set(self): - self.cursor.execute(QU.create_set) - - return self.cursor.fetchone()[0] + 1 - - def create_entry_country(self): - self.cursor.execute(QU.create_country) - - return self.cursor.fetchone()[0] + 1 - def get(self, *args): try: @@ -114,11 +104,8 @@ class Movies(Kodi): self.cursor.execute(QU.update_country, (self.get_country(country),) + args) def add_country(self, *args): - - country_id = self.create_entry_country() - self.cursor.execute(QU.add_country, (country_id,) + args) - - return country_id + self.cursor.execute(QU.add_country, args) + return self.cursor.lastrowid def get_country(self, *args): @@ -130,11 +117,8 @@ class Movies(Kodi): return self.add_country(*args) def add_boxset(self, *args): - - set_id = self.create_entry_set() - self.cursor.execute(QU.add_set, (set_id,) + args) - - return set_id + self.cursor.execute(QU.add_set, args) + return self.cursor.lastrowid def update_boxset(self, *args): self.cursor.execute(QU.update_set, args) diff --git a/jellyfin_kodi/objects/kodi/queries.py b/jellyfin_kodi/objects/kodi/queries.py index 60d326b4..a9660c8a 100644 --- a/jellyfin_kodi/objects/kodi/queries.py +++ b/jellyfin_kodi/objects/kodi/queries.py @@ -44,14 +44,6 @@ create_movie = """ SELECT coalesce(max(idMovie), 0) FROM movie """ -create_set = """ -SELECT coalesce(max(idSet), 0) -FROM sets -""" -create_country = """ -SELECT coalesce(max(country_id), 0) -FROM country -""" create_musicvideo = """ SELECT coalesce(max(idMVideo), 0) FROM musicvideo @@ -319,12 +311,12 @@ add_unique_id_movie_obj = ["{Unique}", "{MovieId}", "movie", "{UniqueId}", "{Pro add_unique_id_tvshow_obj = ["{Unique}", "{ShowId}", "tvshow", "{UniqueId}", "{ProviderName}"] add_unique_id_episode_obj = ["{Unique}", "{EpisodeId}", "episode", "{UniqueId}", "{ProviderName}"] add_country = """ -INSERT INTO country(country_id, name) -VALUES (?, ?) +INSERT INTO country(name) +VALUES (?) """ add_set = """ -INSERT INTO sets(idSet, strSet, strOverview) -VALUES (?, ?, ?) +INSERT INTO sets(strSet, strOverview) +VALUES (?, ?) """ add_set_obj = ["{Title}", "{Overview}"] add_musicvideo = """ From 6fe450da4d9ff9ebcf3fa473b8d56077a2d4c014 Mon Sep 17 00:00:00 2001 From: Chuddah Date: Sun, 23 Feb 2020 18:27:12 +0000 Subject: [PATCH 2/3] Attempt number 2 on a larger dataset; the people table. Note: This has a previously applied optimization of using a local cache. Before: ``` 458570 66.934 0.000 66.934 0.000 {method 'execute' of 'sqlite3.Cursor' objects} 246771 58.075 0.000 58.075 0.000 {method 'fetchone' of 'sqlite3.Cursor' objects} ``` After: ``` 368883 66.220 0.000 66.220 0.000 {method 'execute' of 'sqlite3.Cursor' objects} 157084 58.160 0.000 58.160 0.000 {method 'fetchone' of 'sqlite3.Cursor' objects} ``` Once again, the number of calls to execute and fetchone are reduced but the total time spent executing each is relatively stable. --- jellyfin_kodi/objects/kodi/kodi.py | 9 ++------- jellyfin_kodi/objects/kodi/queries.py | 8 ++------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/jellyfin_kodi/objects/kodi/kodi.py b/jellyfin_kodi/objects/kodi/kodi.py index dd613089..f1182027 100644 --- a/jellyfin_kodi/objects/kodi/kodi.py +++ b/jellyfin_kodi/objects/kodi/kodi.py @@ -40,11 +40,6 @@ class Kodi(object): return self.cursor.fetchone()[0] + 1 - def create_entry_person(self): - self.cursor.execute(QU.create_person) - - return self.cursor.fetchone()[0] + 1 - def create_entry_genre(self): self.cursor.execute(QU.create_genre) @@ -164,8 +159,8 @@ class Kodi(object): def add_person(self, *args): - person_id = self.create_entry_person() - self.cursor.execute(QU.add_person, (person_id,) + args) + self.cursor.execute(QU.add_person, args) + return self.cursor.lastrowid return person_id diff --git a/jellyfin_kodi/objects/kodi/queries.py b/jellyfin_kodi/objects/kodi/queries.py index a9660c8a..e5ca2301 100644 --- a/jellyfin_kodi/objects/kodi/queries.py +++ b/jellyfin_kodi/objects/kodi/queries.py @@ -12,10 +12,6 @@ create_file = """ SELECT coalesce(max(idFile), 0) FROM files """ -create_person = """ -SELECT coalesce(max(actor_id), 0) -FROM actor -""" create_genre = """ SELECT coalesce(max(genre_id), 0) FROM genre @@ -225,8 +221,8 @@ VALUES (?, ?, ?) """ add_file_obj = ["{PathId}", "{Filename}"] add_person = """ -INSERT INTO actor(actor_id, name) -VALUES (?, ?) +INSERT INTO actor(name) +VALUES (?) """ add_people_movie_obj = ["{People}", "{MovieId}", "movie"] add_people_mvideo_obj = ["{People}", "{MvideoId}", "musicvideo"] From e23d2e1f7b921201a015635fca1e25086ad8ce38 Mon Sep 17 00:00:00 2001 From: Chuddah Date: Sun, 23 Feb 2020 19:31:41 +0000 Subject: [PATCH 3/3] Forgot to remove unexecuted code. --- jellyfin_kodi/objects/kodi/kodi.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/jellyfin_kodi/objects/kodi/kodi.py b/jellyfin_kodi/objects/kodi/kodi.py index f1182027..924f3c13 100644 --- a/jellyfin_kodi/objects/kodi/kodi.py +++ b/jellyfin_kodi/objects/kodi/kodi.py @@ -158,12 +158,9 @@ class Kodi(object): self.cursor.executemany(sql, parameters) def add_person(self, *args): - self.cursor.execute(QU.add_person, args) return self.cursor.lastrowid - return person_id - def _get_person(self, *args): try: return self.add_person(*args)