优化筛选程序运行逻辑

src-tauri/Cargo.lock

@@ -744,15 +744,6 @@ version = "0.8.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 
-[[package]]
-name = "crypto"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf1e6e5492f8f0830c37f301f6349e0dac8b2466e4fe89eef90e9eef906cd046"
-dependencies = [
- "crypto-common",
-]
-
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -820,7 +811,7 @@ dependencies = [
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim 0.11.1",
+ "strsim",
  "syn 2.0.66",
 ]
 
@@ -835,12 +826,6 @@ dependencies = [
  "syn 2.0.66",
 ]
 
-[[package]]
-name = "data-encoding"
-version = "2.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
-
 [[package]]
 name = "der"
 version = "0.7.9"
@@ -886,12 +871,6 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "diff"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
-
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -1135,12 +1114,6 @@ dependencies = [
  "rustc_version",
 ]
 
-[[package]]
-name = "file_diff"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31a7a908b8f32538a2143e59a6e4e2508988832d5d4d6f7c156b3cbc762643a5"
-
 [[package]]
 name = "filetime"
 version = "0.2.23"
@@ -3396,21 +3369,6 @@ dependencies = [
  "windows 0.37.0",
 ]
 
-[[package]]
-name = "ring"
-version = "0.17.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
-dependencies = [
- "cc",
- "cfg-if",
- "getrandom 0.2.15",
- "libc",
- "spin 0.9.8",
- "untrusted",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "rsa"
 version = "0.9.6"
@@ -4120,12 +4078,6 @@ dependencies = [
  "unicode-properties",
 ]
 
-[[package]]
-name = "strsim"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
-
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -4366,21 +4318,14 @@ version = "0.0.1"
 dependencies = [
  "anyhow",
  "async-trait",
- "crypto",
- "data-encoding",
- "diff",
- "digest",
- "file_diff",
  "hex",
  "home",
  "lazy_static",
  "regex",
- "ring",
  "rusqlite",
  "serde",
  "serde_json",
  "sha2",
- "strsim 0.10.0",
  "tauri",
  "tauri-build",
  "tauri-plugin-sql",
@@ -4947,12 +4892,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
-[[package]]
-name = "untrusted"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
-
 [[package]]
 name = "url"
 version = "2.5.0"

src-tauri/Cargo.toml

@@ -23,15 +23,8 @@ serde = { version = "1.0", features = ["derive"] }
 tauri = {version = "1.5.2", features = ["api-all"] }
 uuid = "1.8.0"
 regex = "1.10.4"
-crypto = "0.5.1"
-ring = "0.17.8"
-data-encoding = "2.6.0"
 sha2 = "0.10.8"
-digest = "0.10.7"
 hex = "0.4.3"
-file_diff = "1.0.0"
-strsim = "0.10.0"
-diff = "0.1"
 anyhow = "1"
 async-trait = "0.1.80"
 tracing = "0.1.40"

src-tauri/src/self_plugin/tauri_plugin_file/files.rs

@@ -1,15 +1,10 @@
 use hex;
-use ring::digest::{Context, Digest, SHA256};
 use serde::{Deserialize, Serialize, Serializer};
 use sha2::{Digest as OtherDigest, Sha256}; // 确保导入 `Digest`
-use std::io::{self, Read};
 use std::path::{Path, PathBuf};
-use std::result::Result as new_Result;
-use std::{fs, option};
+use std::time::UNIX_EPOCH;
+use std::fs;
 use tauri::command;
-use std::time::{SystemTime, UNIX_EPOCH};
-// use std::result::Result;
-// use tauri::api::file::IntoInvokeHandler;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
@@ -29,17 +24,17 @@ impl Serialize for Error {
 type Result<T> = std::result::Result<T, Error>;
 
 // 提取 /Users/sysadmin/code/rust_project/tauri-app/diff_source/node_modules 中，最后面的数据
-fn extract_last_value<'a>(path: &'a str, value: &'a str) -> &'a str {
+/* fn extract_last_value<'a>(path: &'a str, value: &'a str) -> &'a str {
     if let Some(index) = path.rfind(value) {
         let (content, _) = path.split_at(index);
         content.trim_end_matches('/')
     } else {
         path
     }
-}
+} */
 
 // 过滤
-fn filter_other_directory(path: &str, directories: &[&str]) -> bool {
+/* fn filter_other_directory(path: &str, directories: &[&str]) -> bool {
     // let directories = ["node_modules", ".git", ".obsidian", ".DS_Store"];
     for directory in directories.iter() {
         if extract_last_value(path, directory) != path {
@@ -47,7 +42,7 @@ fn filter_other_directory(path: &str, directories: &[&str]) -> bool {
         }
     }
     true
-}
+} */
 
 // fn read_files_in_directory(directory: &Path, files: &mut Vec<PathBuf>) -> Result<()> {
 //     if let Ok(entries) = fs::read_dir(directory) {
@@ -232,7 +227,6 @@ pub fn calculate_file_hash(file_path: String) -> Result<String> {
     Ok(hash)
 }
 
-
 #[derive(Debug, Serialize, Deserialize)]
 pub struct FileInfos {
     file_path: PathBuf,
@@ -240,6 +234,7 @@ pub struct FileInfos {
     file_type: Option<String>,
     file_size: u64,
     modified_time: Option<u64>, // 时间戳形式
+    creation_time: Option<u64>,
 }
 
 #[command]
@@ -250,18 +245,31 @@ pub fn get_file_info(file_path: String) -> Result<FileInfos> {
     let metadata = fs::metadata(&path)?;
 
     // 获取文件修改时间
-    let modified_time = metadata.modified().ok()
+    let modified_time = metadata
+        .modified()
+        .ok()
+        .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
+        .map(|d| d.as_secs());
+
+    // 获取文件创建时间
+    let accessed_time = metadata
+        .accessed()
+        .ok()
         .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
         .map(|d| d.as_secs());
 
     // 构造FileInfo结构
     let file_info = FileInfos {
         file_path: path.to_path_buf(),
-        file_name: path.file_name().and_then(|name| name.to_str()).map(|name| name.to_string()),
+        file_name: path
+            .file_name()
+            .and_then(|name| name.to_str())
+            .map(|name| name.to_string()),
         file_type: get_file_type(&file_path).map(|t| t.to_string()),
         file_size: metadata.len(),
         modified_time,
+        creation_time: accessed_time,
     };
 
     Ok(file_info)
-}
+}

src/databases/createTableSql.ts

@@ -14,9 +14,12 @@ export const createSql = {
     );`,
     search_files: `CREATE TABLE IF NOT EXISTS search_files (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
-        time TIMESTAMP,
+        create_time TIMESTAMP,
+        creation_time TIMESTAMP,
+        modified_time TIMESTAMP,
+        file_size INTEGER,
         sourceId INTEGER,
-        type TEXT CHECK(length(name) <= 255),
+        type TEXT,
         name TEXT,
         path TEXT,
         hash TEXT,

src/pages/DuplicateFile/CalculateDuplicateFiles.tsx

@@ -10,7 +10,11 @@ import {
 } from "@/services";
 import { useEffect, useState } from "react";
 import { useNavigate, useParams } from "react-router-dom";
-import { FileInfoType, stepsStatusType } from "@/types/files";
+import {
+  FileInfoType,
+  insertSearchFilesPasamsType,
+  stepsStatusType,
+} from "@/types/files";
 import { message } from "@tauri-apps/api/dialog";
 import styles from "./CalculateDuplicateFiles.module.less";
 import File from "@/plugins/tauri-plugin-file/file";
@@ -22,7 +26,11 @@ import {
 } from "@ant-design/icons";
 import { readDir, BaseDirectory } from "@tauri-apps/api/fs";
 import { fileTypeList } from "./config";
-import get_progress_by_sourceId, { get_list_by_sourceid } from "@/services/file-service";
+import get_progress_by_sourceId, {
+  get_list_by_sourceid,
+  updateFileHsah,
+} from "@/services/file-service";
+import { resolve } from "path";
 
 export default function CalculateDuplicateFiles() {
   let { fileId } = useParams();
@@ -31,8 +39,9 @@ export default function CalculateDuplicateFiles() {
   const [current, setCurrent] = useState(1);
   const [percent, setPercent] = useState(85);
   const [stepsStatus, setStepsStatus] = useState<stepsStatusType>({
-    scanDir: "finish",
-    fileOptions: "process",
+    // 'wait' | 'process' | 'finish' | 'error';
+    scanDir: "wait",
+    fileOptions: "wait",
     duplicateFiles: "wait",
     done: "wait",
   });
@@ -40,6 +49,14 @@ export default function CalculateDuplicateFiles() {
     pageInit();
   }, []);
 
+  const waittime = (time = 100) => {
+    return new Promise((resolve) => {
+      setTimeout(() => {
+        resolve(0);
+      }, time);
+    });
+  };
+
   async function pageInit() {
     if (fileId) {
       const [data, errorMsg] = await get_info_by_id(Number.parseInt(fileId));
@@ -69,16 +86,20 @@ export default function CalculateDuplicateFiles() {
     }
   }
   async function scanDirAll() {
- 
     // const aabb = await get_progress_by_sourceId(`${fileId}`);
     // console.log(737373, aabb);
-    
+
     // return
-    
+
     // navigate('/calculate-list/' + fileId)
     if (fileInfo.path) {
       // 扫描目录文件
-
+      setStepsStatus({
+        ...stepsStatus,
+        scanDir: "process",
+      });
+      setCurrent(1);
+      setPercent(0);
       // 排除指定的文件大小、或者筛选所有体量的文件
       const size = []; // 全部为空
 
@@ -89,58 +110,122 @@ export default function CalculateDuplicateFiles() {
         path: fileInfo.path,
         types,
       });
+      setPercent(100);
 
-      console.log(636363, files);
+      // console.log(636363, files);
 
       // 计算文件属性
       if (files.length) {
+        setStepsStatus({
+          ...stepsStatus,
+          scanDir: "finish",
+          fileOptions: "process",
+        });
+        // setCurrent(1)
+        setPercent(0);
         // await files.reduce(async ())
-
+        let fileIndex = -1;
+        let allFilesLength = files.length;
         const result = await files.reduce(
           async (prevPromise: any, currentFile: any) => {
             // 等待上一个 Promise 完成
             await prevPromise;
-            console.log(95, currentFile);
             // 获取文件类型和哈希
-            const type = await File.getType(currentFile);
+            const fileInfo = await File.getInfo(currentFile);
             // const hash = await File.getHash(currentFile);
-            const hash = '';
-            /* 
-              const resInfo = await File.getInfo("/Users/sysadmin/Downloads/google-cloud-cli-455.0.0-darwin-arm.tar.gz")
-              console.log(7373, resInfo);
-              return
-                    Object: {
-                      file_name: "google-cloud-cli-455.0.0-darwin-arm.tar.gz",
-                      file_path: "/Users/sysadmin/Downloads/google-cloud-cli-455.0.0-darwin-arm.tar.gz",
-                      file_size: 119890163,
-                      file_type: "gz",
-                      modified_time: 1701394601,
-                      Object Prototype,
-                    }
-            */
+            const hash = "";
+            fileIndex++;
+            setPercent(Math.floor((fileIndex / allFilesLength) * 100));
+            // await waittime(300);
             return insertSearchFiles({
               // 组装数据
               sourceId: `${fileId}`,
               path: currentFile,
               // type: await File.getType(elm),
-              name: currentFile,
+              name: fileInfo.file_name,
+              creation_time: fileInfo.creation_time,
+              modified_time: fileInfo.modified_time,
+              file_size: fileInfo.file_size,
+              type: fileInfo.file_type,
               hash,
-              type,
             });
           },
           Promise.resolve(0)
         );
-
-        console.log(result); // 顺序处理每个项，然后输出最终结果
+        setPercent(100);
+        await waittime(1000);
         // 计算文件具体内容
-        const allList = await get_list_by_sourceid(`${fileId}`)
-        console.log(137, allList);
-        
+        const [allList, allListMsg] = await get_list_by_sourceid(`${fileId}`);
+        console.log({
+          allList,
+          allListMsg,
+        });
 
+        if (allList) {
+          let fileIndex = -1;
+          let allFilesLength = allList.length;
+          setStepsStatus({
+            ...stepsStatus,
+            scanDir: "finish",
+            fileOptions: "finish",
+            duplicateFiles: "process",
+          });
+          setPercent(0);
+          console.log(173, allFilesLength);
+
+          const allListresult = await allList
+            .filter(
+              (currentFile: insertSearchFilesPasamsType) => !currentFile.hash
+            )
+            .reduce(
+              async (
+                prevPromise: any,
+                currentFile: insertSearchFilesPasamsType
+              ) => {
+                // 等待上一个 Promise 完成
+                await prevPromise;
+                // 获取文件类型和哈希
+                // const type = await File.getType(currentFile);
+                const hash = await File.getHash(currentFile.path);
+                fileIndex++;
+                await waittime();
+                setPercent(Math.floor((fileIndex / allFilesLength) * 100));
+                return updateFileHsah(currentFile.path, hash, `${fileId}`);
+              },
+              Promise.resolve(0)
+            );
+
+          await waittime(1000);
+          setStepsStatus({
+            ...stepsStatus,
+            scanDir: "finish",
+            fileOptions: "finish",
+            duplicateFiles: "finish",
+          });
+          setPercent(100);
+        } else {
+          setStepsStatus({
+            ...stepsStatus,
+            scanDir: "finish",
+            fileOptions: "finish",
+            duplicateFiles: "finish",
+          });
+          setPercent(100);
+          await waittime(2000);
+        }
+
+        setStepsStatus({
+          ...stepsStatus,
+          scanDir: "finish",
+          fileOptions: "finish",
+          duplicateFiles: "finish",
+          done: "process",
+        });
+        setPercent(0);
         // 分析重复文件
-        const searchDuplicateFileRes =  await searchDuplicateFile({
-            sourceId: fileId || ''
-        })
+        const searchDuplicateFileRes = await searchDuplicateFile({
+          sourceId: fileId || "",
+        });
         /* 
             [
                 {count: 6, hash: "3ba7bbfc03e3bed23bf066e2e9a6a5389dd33fd8637bc0220d9e6d642ccf5946", ids: "17,21,22,26,27,31", },
@@ -176,7 +261,16 @@ export default function CalculateDuplicateFiles() {
     
         */
         console.log(747474, searchDuplicateFileRes);
-        if(searchDuplicateFileRes[0]) {}
+        if (searchDuplicateFileRes[0]) {
+        }
+
+        setStepsStatus({
+          scanDir: "finish",
+          fileOptions: "finish",
+          duplicateFiles: "finish",
+          done: "finish",
+        });
+        setPercent(100);
       }
     }
   }
@@ -186,7 +280,6 @@ export default function CalculateDuplicateFiles() {
     if (!fileInfo.checkedTypeValues?.length || !fileInfo.checkedTypeValues)
       return [];
     const checkedTypeValues = `${fileInfo.checkedTypeValues}`?.split(",");
-    console.log(84884, checkedTypeValues);
     fileTypeList.map((elm) => {
       if (checkedTypeValues.indexOf(elm.name) > -1) {
         types = types.concat(elm.valus);

src/services/file-service.ts

@@ -158,18 +158,37 @@ export async function insertSearchFiles({
   type,
   name,
   hash,
+  creation_time,
+  modified_time,
+  file_size,
 }: insertSearchFilesPasamsType) {
   try {
     const DB = await Database.load(`sqlite:files_${sourceId}.db`);
     // 创建表
     await DB.execute(createSql.search_files);
     await DB.execute(
-      "INSERT into search_files (time, sourceId, name,type,path,hash, db_version) VALUES ($1, $2, $3, $4, $5, $6, $7)",
-      [new Date().getTime(), sourceId, path, type, name, hash, "1"]
+      `
+        INSERT into search_files 
+          (create_time, sourceId, name, type, path, hash, creation_time, modified_time, file_size, db_version) 
+        VALUES 
+          ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+      `,
+      [
+        new Date().getTime(),
+        sourceId,
+        name,
+        type,
+        path,
+        hash,
+        creation_time,
+        modified_time,
+        file_size,
+        "1",
+      ]
     );
     return Promise.resolve([true, ""]);
   } catch (err) {
-    console.log(145, err);
+    // console.log(145, err);
     if (err && `${err}`.indexOf("UNIQUE constraint failed") > -1) {
       return Promise.resolve([false, "当前路径重复"]);
     }
@@ -228,7 +247,7 @@ export async function get_list_by_sourceid(
     const DB = await Database.load(`sqlite:files_${sourceId}.db`);
     // 创建表
     await DB.execute(createSql.search_files);
-    const res = await DB.execute(
+    const res = await DB.select(
       "SELECT * FROM search_files WHERE sourceId = $1",
       [sourceId]
     );
@@ -290,8 +309,12 @@ type SearchResult = [boolean, DuplicateFileInfo[] | string | unknown];
 
 export async function searchDuplicateFile({
   sourceId,
+  page = 1,
+  pageSize = 1000,
 }: {
   sourceId: string;
+  page?: number;
+  pageSize?: number;
 }): Promise<SearchResult> {
   try {
     const DB = await Database.load(`sqlite:files_${sourceId}.db`);
@@ -306,7 +329,7 @@ export async function searchDuplicateFile({
        sourceId,
        GROUP_CONCAT(id)    AS ids,
        GROUP_CONCAT(path)  AS paths,
-       GROUP_CONCAT(time)  AS times,
+       GROUP_CONCAT(creation_time)  AS creation_tims,
        COUNT(*)           AS count
 FROM search_files
 WHERE sourceId = $1
@@ -314,14 +337,16 @@ WHERE sourceId = $1
   AND hash != "''"
   AND hash != ""
 GROUP BY hash, sourceId
-HAVING COUNT(*) > 1;
+HAVING COUNT(*) > 1
+ORDER BY [creation_time] ASC
+LIMIT $3 OFFSET ($2 - 1) * $3;
 `,
-      [sourceId]
+      [sourceId, page, pageSize]
     );
     console.log(285, res);
     return Promise.resolve([true, res]);
   } catch (err) {
-    console.log(145, err);
+    // console.log(145, err);
     if (err && `${err}`.indexOf("UNIQUE constraint failed") > -1) {
       return Promise.resolve([false, "当前路径重复"]);
     }
@@ -347,3 +372,32 @@ FROM search_files;`,
 
   return res;
 }
+
+export async function updateFileHsah(
+  path?: string,
+  hash?: string,
+  sourceId?: string
+) {
+  try {
+    const DB = await Database.load(`sqlite:files_${sourceId}.db`);
+    // 创建表
+    await DB.execute(createSql.search_files);
+    const result = await DB.execute(
+      `UPDATE search_files 
+             SET hash = $1
+             WHERE path = $2 and sourceId = $3;`,
+      [
+        hash,
+        path, // 假设 path 变量是预定义的
+        sourceId,
+      ]
+    );
+    return false;
+  } catch (error) {
+    console.log(595959, error);
+    if (error && `${error}`.indexOf("UNIQUE constraint failed") > -1) {
+      return "当前数据格式异常";
+    }
+    return error;
+  }
+}

src/types/files.d.ts

@@ -29,7 +29,10 @@ export type insertSearchFilesPasamsType = {
   // progress: number;
   type: string,
   name: string,
-  hash: string
+  hash: string,
+  file_size: string,
+  creation_time?: string,
+  modified_time?: string,
 };
 
 export type historyListType = {