优化重复文件逻辑

src-tauri/src/self_plugin/tauri_plugin_file/files.rs

@@ -382,3 +382,8 @@ pub fn show_file_in_explorer(file_path: String) -> RequestMvFile {
         },
     }
 }
+
+#[command]
+pub fn file_exists(file_path: &str) -> bool {
+    Path::new(file_path).exists()
+}

src-tauri/src/self_plugin/tauri_plugin_file/mod.rs

@@ -19,8 +19,9 @@ pub fn init<R: Runtime>() -> TauriPlugin<R> {
             get_app_data_dir,
             // open_finder
             show_file_in_explorer,
-            move_specific_files
+            move_specific_files,
             // get_file_info_by_path,
+            file_exists
         ])
         .setup(|_app| {
             // app.manage(SqliteMap::default());

src/components/Menu/Menu.tsx

@@ -13,7 +13,7 @@ export default function Menu() {
   }, []);
 
   async function initMenu() {
-    console.log(141414, location);
+    // console.log(141414, location);
 
     // const config = LogicalSize;
     // console.log({ LogicalSize: new LogicalSize() });

src/pages/DuplicateFile/CalculateDuplicateFiles.tsx

@@ -50,9 +50,6 @@ export default function CalculateDuplicateFiles() {
     // 这段代码只会在组件首次挂载时执行一次
     console.log("组件已挂载");
 
-    // console.log(location); // 当前路由路径
-    // console.log(location.pathname); // 当前路由路径
-
     setTimeout(() => {
       // 设置一个状态标志，表示组件已经挂载
       setHasMounted(true);
@@ -104,7 +101,7 @@ export default function CalculateDuplicateFiles() {
       const files = await scanAllFilesInDir();
 
       // 计算文件属性
-      console.log("计算文件属性 开始");
+      console.log("计算文件属性 开始", files);
       await computeFileMetadata_v2(files);
       console.log("计算文件属性 结束");
 
@@ -148,7 +145,8 @@ export default function CalculateDuplicateFiles() {
   // 扫描目录文件
   async function scanAllFilesInDir(): Promise<backFileInfoType[]> {
     const [progressRes] = await get_progress_by_sourceId(`${fileId}`);
-    if (progressRes.total_entries !== fileInfo.files || !fileInfo.files) {
+    console.log(148, progressRes, fileInfo);
+    if (progressRes.total_entries !== fileInfo.files || !fileInfo.files || (progressRes.hash_null_count && progressRes.total_entries) ) {
       console.log("扫描目录文件 开始");
       setStepsStatus({
         ...stepsStatus,
@@ -177,6 +175,7 @@ export default function CalculateDuplicateFiles() {
    * */
   async function computeFileMetadata_v2(files: backFileInfoType[]) {
     const [progressRes] = await get_progress_by_sourceId(`${fileId}`);
+    console.log(176, files, progressRes);
     // if (!files.length || (!progressRes.total_entries && !progressRes.hash_null_count)) {
     if (!files.length || progressRes.hash_null_count) {
       setStepsStatus({
@@ -203,6 +202,12 @@ export default function CalculateDuplicateFiles() {
       fileIndex++;
       const file_info = files[fileIndex];
       setPercent(Math.floor((fileIndex / allFilesLength) * 100));
+      // 如果已经存在数据库中，跳过记录
+      const [pathInfo] = await get_fileInfo_by_path(`${file_info.file_path}`, `${fileId}`);
+      if( pathInfo?.name ) {
+        return Promise.resolve(0);
+      }
+      console.log(208, pathInfo);
       return insertSearchFiles({
         // 组装数据
         sourceId: `${fileId}`,
@@ -222,9 +227,8 @@ export default function CalculateDuplicateFiles() {
   // 计算每一个文件的hash
   async function computeFileChecksums_2() {
     const [progressRes] = await get_progress_by_sourceId(`${fileId}`);
-
     // 已经存在的数据中，计算过的 hash 总量跟 文件总数不是一样的，并且存在有记录的文件
-    if (progressRes.hash_null_count && progressRes.total_entries) {
+    if (progressRes.hash_null_count && progressRes.total_entries || (!progressRes.hash_null_count && progressRes.total_entries > 0)) {
       let fileIndex = -1;
       let allFilesLength = progressRes.hash_null_count;
       const allList = [...Array(allFilesLength).keys()];
@@ -238,32 +242,44 @@ export default function CalculateDuplicateFiles() {
       await allList.reduce(async (prevPromise: any, index: number) => {
         // 等待上一个 Promise 完成
         await prevPromise;
-        if (
-          isCancelled ||
-          window.location.href.indexOf(location.pathname) < 0
-        ) {
-          // @ts-ignore
-          throw "提前终止";
-          return Promise.resolve(0);
-        } // 如果设置了取消标志，则提前终止
-        const [fileinfo, error] = await getFirstEmptyHashBySourceId(
-          `${fileId}`,
-        );
-        // && fileinfo.file_size / 1024 / 1024 / 1024 < 1 ||
-        if (fileinfo) {
-          // 获取文件类型和哈希
-          const hash = await File.getHash(fileinfo.path);
-          await updateFileHsah(fileinfo.path, hash, `${fileId}`);
+        try {
+          if (
+            isCancelled ||
+            window.location.href.indexOf(location.pathname) < 0
+          ) {
+            // @ts-ignore
+            throw "提前终止";
+            return Promise.resolve(0);
+          } // 如果设置了取消标志，则提前终止
+          const [fileinfo, error] = await getFirstEmptyHashBySourceId(
+            `${fileId}`,
+          );
+          const isHave = await File.isHave(fileinfo.path)
+                // 如果已经存在数据库中，跳过记录
+          const [pathInfo] = await get_fileInfo_by_path(`${fileinfo.file_path}`, `${fileId}`);
+          if( pathInfo?.hash ) {
+            return Promise.resolve(0);
+          }
+          // && fileinfo.file_size / 1024 / 1024 / 1024 < 1 ||
+          if (fileinfo && isHave) {
+            // 获取文件类型和哈希
+            const hash = await File.getHash(fileinfo.path);
+            await updateFileHsah(fileinfo.path, hash, `${fileId}`);
+          }
+          fileIndex++;
+          // await waittime();
+          const [newProgressRes] = await get_progress_by_sourceId(`${fileId}`);
+          setPercent(
+            Math.floor((fileIndex / newProgressRes.hash_null_count) * 100),
+          );
+          
+          setDuplicateFilesStep(
+            `: ${fileIndex} / ${newProgressRes.hash_null_count}`,
+          );
+          console.log(263, `: ${fileIndex} / ${newProgressRes.hash_null_count}`,)
+        } catch (err ) {
+          console.log('computeFileChecksums_2',err)
         }
-        fileIndex++;
-        // await waittime();
-        const [newProgressRes] = await get_progress_by_sourceId(`${fileId}`);
-        setPercent(
-          Math.floor((fileIndex / newProgressRes.hash_null_count) * 100),
-        );
-        setDuplicateFilesStep(
-          `: ${fileIndex} / ${newProgressRes.hash_null_count}`,
-        );
         return Promise.resolve(0);
       }, Promise.resolve(0));
       setDuplicateFilesStep('');

src/pages/DuplicateFile/CalculateListPage.tsx

@@ -10,28 +10,29 @@ import {
   InputRef,
   Spin,
 } from "antd";
-import React, { useEffect, useState, useRef } from "react";
-import { SearchOutlined, FolderOpenOutlined } from "@ant-design/icons";
+import React, {useEffect, useState, useRef} from "react";
+import {SearchOutlined, FolderOpenOutlined} from "@ant-design/icons";
 import {
   message as tauriMessage,
   save as dialogSave,
   open as dialogOpen,
 } from "@tauri-apps/api/dialog";
-import { insertSearchFilesPasamsType } from "@/types/files";
+import {insertSearchFilesPasamsType} from "@/types/files";
 import dayjs from "dayjs";
-import { useParams } from "react-router";
-import { useLocation } from "react-router-dom";
-import { homeDir } from "@tauri-apps/api/path";
+import {useParams} from "react-router";
+import {useLocation} from "react-router-dom";
+import {homeDir} from "@tauri-apps/api/path";
 import File from "@/plugins/tauri-plugin-file/file";
-import { CopyText } from "@/components/Table/CopyText";
-import { formatFileSize } from "@/utils";
+import {CopyText} from "@/components/Table/CopyText";
+import {formatFileSize} from "@/utils";
 import styles from "./CalculateListPage.module.less";
 import {
   del_file_by_id,
   get_fileInfo_by_id,
   searchDuplicateFile,
   setDuplicateFile,
-  getDuplicateFiles_v2,
+  getDuplicateFiles_v2, getDuplicateFilesInfo, updateDuplicateFilesInfo, get_fileInfo_by_path,
+  resetDuplicateFiles,
 } from "@/services";
 
 type KeywordsType<T> = {
@@ -39,7 +40,7 @@ type KeywordsType<T> = {
 };
 
 export default function CalculateListPage() {
-  let { fileId } = useParams();
+  let {fileId} = useParams();
   const location = useLocation();
   const searchInput = useRef<InputRef>(null as any);
   const [data, setData] = useState<FileItem[]>([]);
@@ -55,8 +56,10 @@ export default function CalculateListPage() {
   const [isCancelled, setIsCancelled] = useState(false); // 离开页面时终止正在执行的逻辑
   const [hasMounted, setHasMounted] = useState(false);
   const [spinTip, setSpinTip] = useState("");
+  const [userSelectedRows, setUserSelectedRows] = useState<any>([]);
   //
   const [settimeStart, setSettimeStart] = useState<NodeJS.Timeout>();
+
   interface FileItem {
     sourceId: number;
     ids: string;
@@ -68,18 +71,19 @@ export default function CalculateListPage() {
   }
 
   useEffect(() => {
-    appendData();
+    // appendData();
   }, []);
 
   useEffect(() => {
-    if (hasMounted) {
+    /*if (hasMounted) {
       getFileList();
     } else {
       const startTime: NodeJS.Timeout = setTimeout(() => {
         getFileList();
       }, 5000);
       setSettimeStart(startTime);
-    }
+    }*/
+    getFileList();
   }, [page, pageSize, sorterColumnKey, sorterOrder]);
 
   useEffect(() => {
@@ -102,7 +106,8 @@ export default function CalculateListPage() {
   function stopStartFn() {
     clearTimeout(settimeStart);
   }
-  async function getFileList() {
+
+  async function getFileList(line?: string) {
     stopStartFn();
     setLoading(true);
     try {
@@ -135,7 +140,7 @@ export default function CalculateListPage() {
       const newData = await processFiles(res.data, fileId);
 
       setTimeout(() => {
-        console.log(100, newData);
+        // console.log(100, newData);
         setData(newData);
         setSpinTip("");
         setLoading(false);
@@ -157,15 +162,15 @@ export default function CalculateListPage() {
     }
     const results = [];
     for (const file of data) {
+      const ids = file.ids.split(",");
       const otherItems = await Promise.allSettled(
-        file.ids
-          .split(",")
+        ids
           .filter((elm: any, index: any) => index)
           .map((id: string) => get_fileInfo_by_id(id, fileId)),
       );
       results.push({
         ...file,
-        key: file.id,
+        key: `${file.id}_${ids[0]}`,
         children: (otherItems as any[])
           .filter(
             (result: any) => result.status === "fulfilled" && !result.value[1],
@@ -180,19 +185,27 @@ export default function CalculateListPage() {
     return results;
   }
 
-  const appendData = async () => {
+  const appendData = async (type?: string) => {
+    if(type === 'reset') {
+      await resetDuplicateFiles(`${fileId}`)
+    }
     setLoading(true);
     setRemoveList([]);
     const [isError, searchDuplicateFileRes] = await searchDuplicateFile({
       sourceId: `${fileId}`,
+      type: `${type}`
     });
-    if (!isError) {
+    /* if (!isError) {
       typeof searchDuplicateFileRes === "string" &&
-        (await tauriMessage(searchDuplicateFileRes, {
-          title: "查询失败",
-          type: "error",
-        }));
-    }
+      (await tauriMessage(searchDuplicateFileRes, {
+        title: "查询失败",
+        type: "error",
+      }));
+    } */
+    /* console.log(206,  !searchDuplicateFileRes ||
+      !Array.isArray(searchDuplicateFileRes) ||
+      !searchDuplicateFileRes.length);
+    
     if (
       !searchDuplicateFileRes ||
       !Array.isArray(searchDuplicateFileRes) ||
@@ -202,7 +215,7 @@ export default function CalculateListPage() {
       await getFileList();
       setLoading(false);
       return;
-    }
+    } */
     /**
      * count: 2
      * hash: "fdd8051fcf884d8cc9a095cd77a58694e13b066aea68dc1fc353767ab0ebfe01"
@@ -210,6 +223,9 @@ export default function CalculateListPage() {
      * sourceId: 6
      * */
     setLoading(true);
+    setTimeout(() => {
+      setLoading(false);
+    }, 3000);
     let index = -1;
     await searchDuplicateFileRes.reduce(
       async (prevPromise: any, currentFile: any) => {
@@ -221,7 +237,6 @@ export default function CalculateListPage() {
         ) {
           // @ts-ignore
           throw "提前终止";
-          return Promise.resolve(0);
         } // 如果设置了取消标志，则提前终止
         index++;
         const ids = currentFile.ids.split(",");
@@ -234,7 +249,7 @@ export default function CalculateListPage() {
             idsNum: ids.length,
           });
         } catch (err) {
-          console.log(199, err);
+          console.log(199, err); 
         }
         setSpinTip(
           `进度: ${Math.floor((index / searchDuplicateFileRes.length) * 100)}% / 剩余 ${searchDuplicateFileRes.length - index} 个文件`,
@@ -263,26 +278,19 @@ export default function CalculateListPage() {
     setLoading(true);
     const filesRes = await Promise.allSettled(
       removeList.map((path) => File.rmFile(path)),
+      /* removeList.map((path) => Promise.resolve({
+        code: 200,
+        msg: 'File successfully moved to trash.',
+        data: path
+      })), */
     );
-    if (removeList.length === 1) {
-      if (
-        filesRes[0].status === "fulfilled" &&
-        (filesRes[0] as any).value.code === 200
-      ) {
-        setRemoveList([]);
-        await del_file_by_id(removeList[0], `${fileId}`);
-        message.success(`${removeList[0]} 删除成功!`);
-        await appendData();
-        return;
+    const rmSuccess = filesRes.map((res) => {
+      if (res.status === "fulfilled" && res.value.code === 200) {
+        return res.value.data
       }
-      await tauriMessage(removeList[0], {
-        title: "删除失败",
-        type: "error",
-      });
-    }
-    const rmSuccess = filesRes.filter((res) => {
-      return res.status === "fulfilled" && res.value.code === 200;
-    });
+      return false
+    }).filter(res => res);
+    console.log(299, rmSuccess)
     if (rmSuccess.length) {
       await rmSuccess.reduce(async (prevPromise: any, item: any) => {
         await prevPromise;
@@ -293,13 +301,28 @@ export default function CalculateListPage() {
           // @ts-ignore
           throw "提前终止";
           return Promise.resolve(0);
-        } // 如果设置了取消标志，则提前终止
-        return del_file_by_id(item.value.data, `${fileId}`);
+        }
+        // 如果设置了取消标志，则提前终止
+        // 删除 duplicate_files 中的数据, 先找到
+        const idInfoIndex = userSelectedRows.findIndex((elm: any) => elm.path === item);
+        if (idInfoIndex > -1) {
+          const idInfo = userSelectedRows[idInfoIndex];
+          if (`${idInfo.key}`.indexOf('_') > -1) {
+            const [id, sourceId] = idInfo.key.split('_');
+            const [row] = await getDuplicateFilesInfo(`${fileId}`, id);
+            const [fileInfo_by_path] = await get_fileInfo_by_path(item, `${fileId}`)
+            const ids = row.ids.split(',').filter((i: any) => Number.parseInt(i) !== Number.parseInt(fileInfo_by_path.id));
+            await del_file_by_id(item, `${fileId}`);
+            await updateDuplicateFilesInfo(ids.toString(), ids.length, id, `${fileId}`);
+          }
+        }
+        return Promise.resolve()
       }, Promise.resolve());
       message.success(
         `${rmSuccess.length}个文件删除成功! ${filesRes.length - rmSuccess.length}个文件删除失败!`,
       );
-      await appendData();
+      // await appendData();
+      getFileList()
       await waittime(1500);
       setLoading(false);
       return;
@@ -360,29 +383,29 @@ export default function CalculateListPage() {
 
   const getColumnSearchProps = (dataIndex: string, dataIndexName: string) => ({
     filterDropdown: () => (
-      <div style={{ padding: 8 }} onKeyDown={(e) => e.stopPropagation()}>
+      <div style={{padding: 8}} onKeyDown={(e) => e.stopPropagation()}>
         <Input
           ref={searchInput}
           placeholder={`搜索${dataIndexName}`}
           value={searchText[dataIndex]}
           onChange={(e) => handleSearch(dataIndex, e.target.value)}
           onPressEnter={() => getFileList()}
-          style={{ marginBottom: 8, display: "block" }}
+          style={{marginBottom: 8, display: "block"}}
         />
         <Space>
           <Button
             type="primary"
-            icon={<SearchOutlined />}
+            icon={<SearchOutlined/>}
             size="small"
             onClick={() => getFileList()}
-            style={{ width: 90 }}
+            style={{width: 90}}
           >
             搜索
           </Button>
           <Button
             size="small"
             onClick={() => handleSearchReset(dataIndex)}
-            style={{ width: 90 }}
+            style={{width: 90}}
           >
             重置
           </Button>
@@ -435,6 +458,7 @@ export default function CalculateListPage() {
       dataIndex: "idsNum",
       key: "idsNum",
       sorter: true,
+      width: '200px'
     },
     {
       title: "文件大小",
@@ -508,6 +532,11 @@ export default function CalculateListPage() {
         selectedRows,
       );
       setUserSelectedRowKeys(selectedRowKeys);
+      setUserSelectedRows(selectedRows);
+      setRemoveList(selectedRows.map((elm: any) => {
+        // console.log(514, );
+        return elm.path
+      }))
     },
     onSelect: (record: any, selected: any, selectedRows: any) => {
       console.log("onSelect", record, selected, selectedRows);
@@ -557,7 +586,7 @@ export default function CalculateListPage() {
           [sorter.columnKey]: sorter.order === "ascend" ? "ASC" : "DESC",
         });
       } else {
-        const _sorterColumnKey = { ...sorterColumnKey };
+        const _sorterColumnKey = {...sorterColumnKey};
         delete _sorterColumnKey[sorter.columnKey];
         setSorterColumnKey({
           ..._sorterColumnKey,
@@ -570,11 +599,17 @@ export default function CalculateListPage() {
     <Spin spinning={loading} tip={spinTip}>
       <div className={styles.CalculateListPage}>
         <Space>
+          <Button type="primary" onClick={() => appendData()}>
+            开始
+          </Button>
+          <Button type="primary" onClick={() => appendData('reset')}>
+            重置
+          </Button>
           <Button type="primary" danger onClick={() => removeFilesByDB()}>
-            删除选中的文件
+            删除文件
           </Button>
           <Button type="primary" onClick={() => openDialogSave()}>
-            统一移动到指定目录
+            迁移文件
           </Button>
           {!loading && spinTip && <div>{spinTip}</div>}
           {/*<Button type="primary">导出</Button>*/}
@@ -590,7 +625,7 @@ export default function CalculateListPage() {
             current: page,
             showQuickJumper: true,
           }}
-          rowSelection={{ ...rowSelection }}
+          rowSelection={{...rowSelection}}
           expandable={{
             defaultExpandAllRows: false,
           }}

src/plugins/tauri-plugin-file/file.ts

@@ -56,6 +56,11 @@ export class File {
       destDir
     });
   }
+  static async isHave(filePath: string): Promise<string> {
+    return await invoke<string>("plugin:st-files|file_exists", {
+      filePath
+    });
+  }
 
   // async close(): Promise<boolean> {
   //     return await invoke('plugin:st-sqlite|close', { path: this.path })

src/services/file-service.ts

@@ -319,13 +319,11 @@ type SearchResult = [boolean, DuplicateFileInfo[] | string | unknown];
 
 export async function searchDuplicateFile({
   sourceId,
-  page = 1,
-  pageSize = 1000,
+  type
 }: {
   sourceId: string;
-  page?: number;
-  pageSize?: number;
-}): Promise<SearchResult> {
+  type: string
+}): Promise<any> {
   try {
     const DB = await Database.load(`sqlite:files_${sourceId}.db`);
     // 创建表
@@ -344,7 +342,7 @@ export async function searchDuplicateFile({
     GROUP_CONCAT(s.id) AS ids,
     COUNT(*) AS count
 FROM search_files s
-LEFT JOIN duplicate_files d ON s.hash = d.hash
+LEFT JOIN duplicate_files d ON s.hash = d.hash ${type === 'reset' && "AND (',' || d.ids || ',' NOT LIKE '%,' || s.id || ',%')"}
 WHERE s.sourceId = $1
   AND s.hash IS NOT NULL
   AND s.hash != ''
@@ -352,8 +350,11 @@ WHERE s.sourceId = $1
 GROUP BY s.hash, s.sourceId
 HAVING COUNT(*) > 1;
 `,
-      [sourceId, page, pageSize],
+// AND (',' || d.ids || ',' NOT LIKE '%,' || s.id || ',%')
+      [sourceId],
     );
+    console.log(353, res);
+    
     return Promise.resolve([true, res]);
   } catch (err) {
     if (err && `${err}`.indexOf("UNIQUE constraint failed") > -1) {
@@ -422,7 +423,7 @@ export async function get_fileInfo_by_id(
       "SELECT * FROM search_files WHERE id = $1 and sourceId = $2",
       [id, sourceId],
     );
-    if (Array.isArray(res)) {
+    if (Array.isArray(res) && res.length) {
       return [res[0], ""];
     }
     return [false, "暂无数据"];
@@ -457,10 +458,11 @@ export async function get_fileInfo_by_path(path: string, sourceId: string) {
 
 export async function del_file_by_id(path: string, sourceId: string) {
   try {
+    console.log('删除的 path', path)
     const DB = await Database.load(`sqlite:files_${sourceId}.db`);
     // 创建表
     await DB.execute(createSql.search_files);
-    await DB.execute(
+    const res = await DB.execute(
       `DELETE FROM search_files WHERE path = $1 and sourceId = $2`,
       [
         path, // 假设 path 变量是预定义的
@@ -727,4 +729,76 @@ export async function duplicateFilesDBInit(sourceId: string) {
 export async function closeDB(sourceId: string) {
   const DB = await Database.load(`sqlite:files_${sourceId}.db`);
   await DB.close();
-}
+}
+
+
+
+
+export async function getDuplicateFilesInfo(sourceId: string, id: string) {
+  try {
+    console.log(`getDuplicateFilesInfo::: ${sourceId} __ ${id}`);
+    const DB = await Database.load(`sqlite:files_${sourceId}.db`);
+    // 创建表
+    await DB.execute(createSql.duplicate_files);
+
+    const res = await DB.select(
+        "SELECT * FROM duplicate_files WHERE id = $1",
+        [id],
+    );
+    if (Array.isArray(res)) {
+      return [res[0], ""];
+    }
+    return [false, "暂无数据"];
+  } catch (error) {
+    if (error && `${error}`.indexOf("UNIQUE constraint failed") > -1) {
+      return [false, "当前路径重复"];
+    }
+    return [false, `${error}`];
+  }
+}
+
+export async function updateDuplicateFilesInfo(
+    ids: string,
+    idsNum: number,
+    id: string,
+    sourceId?: string,
+) {
+  try {
+    const DB = await Database.load(`sqlite:files_${sourceId}.db`);
+    // 创建表
+    await DB.execute(createSql.search_files);
+    await DB.execute(
+        `UPDATE duplicate_files 
+             SET ids = $1, idsNum = $2
+             WHERE id = $3`,
+        [
+          ids,
+          idsNum,
+          id,
+        ],
+    );
+    return false;
+  } catch (error) {
+    if (error && `${error}`.indexOf("UNIQUE constraint failed") > -1) {
+      return "当前数据格式异常";
+    }
+    return error;
+  }
+}
+
+
+export async function resetDuplicateFiles(
+  sourceId?: string,
+) {
+  try {
+    const DB = await Database.load(`sqlite:files_${sourceId}.db`);
+    // 删除表中的所有数据
+    await DB.execute(`DELETE FROM duplicate_files`);
+    return false;
+  } catch (error) {
+    if (error && `${error}`.indexOf("UNIQUE constraint failed") > -1) {
+      return "当前数据格式异常";
+    }
+    return error;
+  }
+}