[Files Refactor] Performance tuning (#2813)

* Do database txn in same thread. Retry on locked db
* Remove captions from slimscenedata
* Fix tracing
* Use where in instead of individual selects
* Remove scenes_query view
* Remove image query view
* Remove gallery query view
* Use where in for FindMany
* Don't interrupt scanning zip files
* Fix image filesize sort

pkg/sqlite/migrations/32_files.up.sql

@@ -539,139 +539,3 @@ ALTER TABLE `scenes_new` rename to `scenes`;
 CREATE INDEX `index_scenes_on_studio_id` on `scenes` (`studio_id`);
 
 PRAGMA foreign_keys=ON;
-
--- create views to simplify queries
-
-CREATE VIEW `images_query` AS 
-  SELECT 
-    `images`.`id`,
-    `images`.`title`,
-    `images`.`rating`,
-    `images`.`organized`,
-    `images`.`o_counter`,
-    `images`.`studio_id`,
-    `images`.`created_at`,
-    `images`.`updated_at`,
-    `galleries_images`.`gallery_id`,
-    `images_tags`.`tag_id`,
-    `performers_images`.`performer_id`,
-    `image_files`.`format` as `image_format`,
-    `image_files`.`width` as `image_width`,
-    `image_files`.`height` as `image_height`,
-    `files`.`id` as `file_id`,
-    `files`.`basename`,
-    `files`.`size`,
-    `files`.`mod_time`,
-    `files`.`zip_file_id`,
-    `folders`.`id` as `parent_folder_id`,
-    `folders`.`path` as `parent_folder_path`,
-    `zip_files`.`basename` as `zip_basename`,
-    `zip_files_folders`.`path` as `zip_folder_path`,
-    `files_fingerprints`.`type` as `fingerprint_type`,
-    `files_fingerprints`.`fingerprint`
-  FROM `images`
-  LEFT JOIN `performers_images` ON (`images`.`id` = `performers_images`.`image_id`) 
-  LEFT JOIN `galleries_images` ON (`images`.`id` = `galleries_images`.`image_id`) 
-  LEFT JOIN `images_tags` ON (`images`.`id` = `images_tags`.`image_id`)
-  LEFT JOIN `images_files` ON (`images`.`id` = `images_files`.`image_id`) 
-  LEFT JOIN `image_files` ON (`images_files`.`file_id` = `image_files`.`file_id`) 
-  LEFT JOIN `files` ON (`images_files`.`file_id` = `files`.`id`) 
-  LEFT JOIN `folders` ON (`files`.`parent_folder_id` = `folders`.`id`) 
-  LEFT JOIN `files` AS `zip_files` ON (`files`.`zip_file_id` = `zip_files`.`id`)
-  LEFT JOIN `folders` AS `zip_files_folders` ON (`zip_files`.`parent_folder_id` = `zip_files_folders`.`id`)
-  LEFT JOIN `files_fingerprints` ON (`images_files`.`file_id` = `files_fingerprints`.`file_id`);
-
-CREATE VIEW `galleries_query` AS 
-  SELECT 
-    `galleries`.`id`,
-    `galleries`.`title`,
-    `galleries`.`url`,
-    `galleries`.`date`,
-    `galleries`.`details`,
-    `galleries`.`rating`,
-    `galleries`.`organized`,
-    `galleries`.`studio_id`,
-    `galleries`.`created_at`,
-    `galleries`.`updated_at`,
-    `galleries_tags`.`tag_id`,
-    `scenes_galleries`.`scene_id`,
-    `performers_galleries`.`performer_id`,
-    `galleries_folders`.`id` as `folder_id`,
-    `galleries_folders`.`path` as `folder_path`,
-    `files`.`id` as `file_id`,
-    `files`.`basename`,
-    `files`.`size`,
-    `files`.`mod_time`,
-    `files`.`zip_file_id`,
-    `parent_folders`.`id` as `parent_folder_id`,
-    `parent_folders`.`path` as `parent_folder_path`,
-    `zip_files`.`basename` as `zip_basename`,
-    `zip_files_folders`.`path` as `zip_folder_path`,
-    `files_fingerprints`.`type` as `fingerprint_type`,
-    `files_fingerprints`.`fingerprint`
-  FROM `galleries`
-  LEFT JOIN `performers_galleries` ON (`galleries`.`id` = `performers_galleries`.`gallery_id`) 
-  LEFT JOIN `galleries_tags` ON (`galleries`.`id` = `galleries_tags`.`gallery_id`)
-  LEFT JOIN `scenes_galleries` ON (`galleries`.`id` = `scenes_galleries`.`gallery_id`) 
-  LEFT JOIN `folders` AS `galleries_folders` ON (`galleries`.`folder_id` = `galleries_folders`.`id`) 
-  LEFT JOIN `galleries_files` ON (`galleries`.`id` = `galleries_files`.`gallery_id`) 
-  LEFT JOIN `files` ON (`galleries_files`.`file_id` = `files`.`id`) 
-  LEFT JOIN `folders` AS `parent_folders` ON (`files`.`parent_folder_id` = `parent_folders`.`id`) 
-  LEFT JOIN `files` AS `zip_files` ON (`files`.`zip_file_id` = `zip_files`.`id`)
-  LEFT JOIN `folders` AS `zip_files_folders` ON (`zip_files`.`parent_folder_id` = `zip_files_folders`.`id`)
-  LEFT JOIN `files_fingerprints` ON (`galleries_files`.`file_id` = `files_fingerprints`.`file_id`);
-
-CREATE VIEW `scenes_query` AS 
-  SELECT 
-    `scenes`.`id`,
-    `scenes`.`title`,
-    `scenes`.`details`,
-    `scenes`.`url`,
-    `scenes`.`date`,
-    `scenes`.`rating`,
-    `scenes`.`studio_id`,
-    `scenes`.`o_counter`,
-    `scenes`.`organized`,
-    `scenes`.`created_at`,
-    `scenes`.`updated_at`,
-    `scenes_tags`.`tag_id`,
-    `scenes_galleries`.`gallery_id`,
-    `performers_scenes`.`performer_id`,
-    `movies_scenes`.`movie_id`,
-    `movies_scenes`.`scene_index`,
-    `scene_stash_ids`.`stash_id`,
-    `scene_stash_ids`.`endpoint`,
-    `video_files`.`format` as `video_format`,
-    `video_files`.`width` as `video_width`,
-    `video_files`.`height` as `video_height`,
-    `video_files`.`duration`,
-    `video_files`.`video_codec`,
-    `video_files`.`audio_codec`,
-    `video_files`.`frame_rate`,
-    `video_files`.`bit_rate`,
-    `video_files`.`interactive`,
-    `video_files`.`interactive_speed`,
-    `files`.`id` as `file_id`,
-    `files`.`basename`,
-    `files`.`size`,
-    `files`.`mod_time`,
-    `files`.`zip_file_id`,
-    `folders`.`id` as `parent_folder_id`,
-    `folders`.`path` as `parent_folder_path`,
-    `zip_files`.`basename` as `zip_basename`,
-    `zip_files_folders`.`path` as `zip_folder_path`,
-    `files_fingerprints`.`type` as `fingerprint_type`,
-    `files_fingerprints`.`fingerprint`
-  FROM `scenes`
-  LEFT JOIN `performers_scenes` ON (`scenes`.`id` = `performers_scenes`.`scene_id`) 
-  LEFT JOIN `scenes_tags` ON (`scenes`.`id` = `scenes_tags`.`scene_id`)
-  LEFT JOIN `movies_scenes` ON (`scenes`.`id` = `movies_scenes`.`scene_id`)
-  LEFT JOIN `scene_stash_ids` ON (`scenes`.`id` = `scene_stash_ids`.`scene_id`)
-  LEFT JOIN `scenes_galleries` ON (`scenes`.`id` = `scenes_galleries`.`scene_id`) 
-  LEFT JOIN `scenes_files` ON (`scenes`.`id` = `scenes_files`.`scene_id`) 
-  LEFT JOIN `video_files` ON (`scenes_files`.`file_id` = `video_files`.`file_id`) 
-  LEFT JOIN `files` ON (`scenes_files`.`file_id` = `files`.`id`) 
-  LEFT JOIN `folders` ON (`files`.`parent_folder_id` = `folders`.`id`) 
-  LEFT JOIN `files` AS `zip_files` ON (`files`.`zip_file_id` = `zip_files`.`id`)
-  LEFT JOIN `folders` AS `zip_files_folders` ON (`zip_files`.`parent_folder_id` = `zip_files_folders`.`id`)
-  LEFT JOIN `files_fingerprints` ON (`scenes_files`.`file_id` = `files_fingerprints`.`file_id`);