优化重复文件的结果页面的展示流程, 避免内存泄露

src/databases/createTableSql.ts

@@ -26,5 +26,20 @@ export const createSql = {
         hash TEXT,
         db_version TEXT,
         UNIQUE (path)
+    );`,
+    duplicate_files: `CREATE TABLE IF NOT EXISTS duplicate_files (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        create_time TIMESTAMP,
+        creation_time TIMESTAMP,
+        modified_time TIMESTAMP,
+        file_size INTEGER,
+        sourceId INTEGER,
+        type TEXT,
+        name TEXT,
+        path TEXT,
+        hash TEXT,
+        db_version TEXT,
+        ids  TEXT,
+        UNIQUE (hash)
     );`
 }

src/pages/DuplicateFile/CalculateListPage.tsx

@@ -16,6 +16,8 @@ import {
   del_file_by_id,
   get_fileInfo_by_id,
   searchDuplicateFile,
+  setDuplicateFile,
+  getDuplicateFile
 } from "@/services";
 import {
   message as tauriMessage,
@@ -30,13 +32,18 @@ import type { GetProp } from "antd";
 import File from "@/plugins/tauri-plugin-file/file";
 import { CopyText } from "@/components/Table/CopyText";
 import { FolderOpenOutlined } from "@ant-design/icons";
+import VirtualList from 'rc-virtual-list';
+import dayjs from 'dayjs'
+import { formatFileSize } from '@/utils';
 
 export default function CalculateListPage() {
+  const ContainerHeight = 500;
   let { fileId } = useParams();
   const [data, setData] = useState<FileItem[]>([]);
   const [loading, setLoading] = useState<boolean>(false);
   const [tip, setTip] = useState<string>('');
   const [removeList, setRemoveList] = useState<string[]>([]);
+  const [page, setPage] = useState(1)
   interface FileItem {
     sourceId: number;
     ids: string;
@@ -45,47 +52,20 @@ export default function CalculateListPage() {
     firstItem: insertSearchFilesPasamsType;
     otherItems: insertSearchFilesPasamsType[];
   }
-  const appendData = async () => {
-    setLoading(true)
-    setTip('正在统计中');
-    const [isError, searchDuplicateFileRes] = await searchDuplicateFile({
+
+  async function getFileList() {
+    setLoading(true);
+    setTip('加载数据中~')
+    const res = await getDuplicateFile({
       sourceId: `${fileId}`,
-    });
-    console.log(5151, isError)
-    if (!isError) {
-      typeof searchDuplicateFileRes === "string" &&
-        (await tauriMessage(searchDuplicateFileRes, {
-          title: "查询失败",
-          type: "error",
-        }));
-    }
-    /*count
-        :
-        2
-    hash
-        :
-        "fdd8051fcf884d8cc9a095cd77a58694e13b066aea68dc1fc353767ab0ebfe01"
-    ids
-        :
-        "25494,26393"
-    sourceId
-        :
-        6*/
-    setTip('');
-    setLoading(false);
-    setData(searchDuplicateFileRes as any);
-    console.log(63, searchDuplicateFileRes);
-    return
-    if (Array.isArray(searchDuplicateFileRes)) {
-      let index = -1
-      const newData: any[] = [];
-      await searchDuplicateFileRes.reduce(
+       page,
+      pageSize: 10
+    })
+    const newData: any[] = [];
+    if(Array.isArray(res.data)) {
+      await res.data.reduce(
         async (prevPromise: any, currentFile: any) => {
-          // 等待上一个 Promise 完成
-          await prevPromise;
-          index++
           const ids = currentFile.ids.split(",");
-          const firstItem = await get_fileInfo_by_id(ids[0], `${fileId}`);
           const otherItems = await Promise.allSettled(
             ids
               .map((id: string) => {
@@ -96,29 +76,86 @@ export default function CalculateListPage() {
               })
               .filter((elm: any) => elm)
           );
-
           newData.push({
             ...currentFile,
-            firstItem: firstItem[0],
             otherItems: otherItems
               .map((elm) => {
                 if (elm.status === "fulfilled" && !elm.value[1]) {
-                  setRemoveList([]);
                   return elm.value[0];
                 }
                 return false;
               })
               .filter((elm: any) => elm),
           });
+          return Promise.resolve(0)
+        }
+      ), Promise.resolve(0)
+      const allFilesHash = data.map(elm => elm.hash);
+      const noDuplicateData = newData.filter(elm => {
+        if(allFilesHash.indexOf(elm.hash) > -1) {
+          return false
+        }
+        return elm
+      })
+      setData([
+        ...data,
+        ...noDuplicateData
+      ]);
+      setTimeout(() => {
+        setLoading(false);
+        setTip('');
+      }, 300)
+    }
+  }
+  const appendData = async () => {
+    setLoading(true);
+    setRemoveList([]);
+    setTip('正在统计中');
+    const [isError, searchDuplicateFileRes] = await searchDuplicateFile({
+      sourceId: `${fileId}`,
+    });
+    if (!isError) {
+      typeof searchDuplicateFileRes === "string" &&
+        (await tauriMessage(searchDuplicateFileRes, {
+          title: "查询失败",
+          type: "error",
+        }));
+    }
+    /*
+      count: 2
+      hash: "fdd8051fcf884d8cc9a095cd77a58694e13b066aea68dc1fc353767ab0ebfe01"
+      ids: "25494,26393"
+      sourceId: 6
+    */
+    if (Array.isArray(searchDuplicateFileRes)) {
+      let index = -1
+      await searchDuplicateFileRes.reduce(
+        async (prevPromise: any, currentFile: any) => {
+          // 等待上一个 Promise 完成
+          await prevPromise;
+          index++
+          const ids = currentFile.ids.split(",");
+          const firstItem = await get_fileInfo_by_id(ids[0], `${fileId}`);
+          try {
+            // 写到本地的数据库中
+            await setDuplicateFile(`${fileId}`, {
+              ...firstItem[0],
+              ids: currentFile.ids
+            })
+          } catch (err) {
+            console.log(109, err);
+          }
           setTip(`正在统计中: ${Math.floor((index / searchDuplicateFileRes.length) * 100)}% : ${searchDuplicateFileRes.length - index}`);
           return Promise.resolve(0);
         },
         Promise.resolve(0)
       );
-      setData(newData);
+      // 执行可分页的接口数据请求
+      await getFileList();
+    } else {
+      setLoading(false)
+      setTip('')
     }
-    setLoading(false)
-    setTip('')
   };
 
   useEffect(() => {
@@ -156,20 +193,18 @@ export default function CalculateListPage() {
         ></CopyText>
       </div>
       <div className={styles.modified_time}>
-        {/* <CopyText
-          width="100px"
+        <CopyText
+          width="150px"
           color="#333"
-          name={item.modified_time || ""}
-        ></CopyText> */}
-        {item.modified_time}
+          name={item.modified_time ? dayjs.unix(item.modified_time).format('YYYY-MM-DD HH:mm:ss') : ""}
+        ></CopyText>
       </div>
       <div className={styles.modified_time}>
-        {/* <CopyText
+        <CopyText
           width="100px"
           color="#333"
-          name={item.file_size || ""}
-        ></CopyText> */}
-        {item.file_size}
+          name={item.file_size ? formatFileSize(item.file_size) : ''}
+        ></CopyText>
       </div>
     </div>
   );
@@ -235,7 +270,7 @@ export default function CalculateListPage() {
       multiple: false,
       defaultPath: await homeDir(),
     });
-    console.log(213, selected);
+
     await File.moveSpecificFiles(['/Users/honghaitao/Downloads/Xnip2023-05-31_21-39-11_副本.png'], `${selected}`)
     return;
     // dialogSave
@@ -247,7 +282,14 @@ export default function CalculateListPage() {
         },
       ],
     });
-    console.log(186, filePath);
+  }
+  useEffect(() => {
+    if(data.length) {
+      getFileList();  
+    }
+  }, [page])
+  function getNext() {
+    setPage(page + 1);
   }
   return (
     <div className={styles.CalculateListPage}>
@@ -274,47 +316,53 @@ export default function CalculateListPage() {
             value={removeList}
           >
             <div style={{ width: "100%" }}>
-              {data.map((item: any) => (
-                <div
-                  key={item.hash}
-                  style={{
-                    backgroundColor: "var(--color-2)",
-                    marginBottom: "24px",
-                  }}
+              <List>
+                <VirtualList
+                    data={data}
+                    itemKey="hash"
                 >
-                  <div className={styles.CheckboxGroup}>
-                    <Checkbox value={item.path}>
-                      {/*{CheckboxContent(item as any)}*/}
-                      {item.path}
-                    </Checkbox>
-                  </div>
-                  <div
-                    style={{
-                      border: "1px solid var(--color-1)",
-                      padding: "12px 3px",
-                    }}
-                    className={styles.CheckboxGroup}
-                  >
-                    {/*{item.otherItems.map((otherItem) => (
-                      <div key={otherItem.path}>
-                        <Checkbox value={otherItem.path}>
-                          {CheckboxContent(otherItem)}
-                        </Checkbox>
-                      </div>
-                    ))}*/}
-                    {item.ids.split(',').map((id_name: string) => (
-                      <div key={id_name}>
-                        <Checkbox value={id_name}>
-                          {/*{CheckboxContent(id_name as any)}*/}
-                          {id_name}
-                        </Checkbox>
+                {/*height={ContainerHeight}*/}
+                  {(item: any) => (
+                    <List.Item key={item.hash}>
+                      <div
+                        key={item.hash}
+                        style={{
+                          backgroundColor: "var(--color-2)",
+                          marginBottom: "24px",
+                          width: '100%'
+                        }}
+                      >
+                        <div className={styles.CheckboxGroup}>
+                          <Checkbox value={item.path}>
+                            {CheckboxContent(item as any)}
+                            {/*item.path*/}
+                          </Checkbox>
+                        </div>
+                        <div
+                          style={{
+                            border: "1px solid var(--color-1)",
+                            padding: "12px 3px",
+                          }}
+                          className={styles.CheckboxGroup}
+                        >
+                          {item.otherItems.map((otherItem) => (
+                            <div key={otherItem.path}>
+                              <Checkbox value={otherItem.path}>
+                                {CheckboxContent(otherItem)}
+                              </Checkbox>
+                            </div>
+                          ))}
+                        </div>
                       </div>
-                    ))}
-                  </div>
-                </div>
-              ))}
+                    </List.Item>
+                  )}
+                </VirtualList>
+              </List>
             </div>
           </Checkbox.Group>
+          <Button type="primary" onClick={() => getNext()}>
+            加载更多
+          </Button>
           {!data.length && !loading && (
             <div
               style={{

src/services/file-service.ts

@@ -340,32 +340,28 @@ export async function searchDuplicateFile({
     const DB = await Database.load(`sqlite:files_${sourceId}.db`);
     // 创建表
     await DB.execute(createSql.search_files);
-    /*
-    select * from search_files where sourceId = $1 in (select sourceId from search_files group by hash having count(hash) > 1)
- */
-    // const res = await DB.select("SELECT * from search_files WHERE sourceId = $1", [sourceId]);
     const res: DuplicateFileInfo[] = await DB.select(
-      `SELECT hash,
-       sourceId,
-       id,
-       creation_time,
-       modified_time,
-       file_size,
-       type,
-       name,
-       path,
-       GROUP_CONCAT(id)    AS ids,
-       COUNT(*)           AS count
-FROM search_files
-WHERE sourceId = $1
-  AND hash IS NOT NULL  
-  AND hash != "''"
-  AND hash != ""
-GROUP BY hash, sourceId
-HAVING COUNT(*) > 1
+      `SELECT
+    s.hash,
+    s.sourceId,
+    s.id,
+    s.creation_time,
+    s.modified_time,
+    s.file_size,
+    s.type,
+    s.name,
+    s.path,
+    GROUP_CONCAT(s.id) AS ids,
+    COUNT(*) AS count
+FROM search_files s
+LEFT JOIN duplicate_files d ON s.hash = d.hash
+WHERE s.sourceId = $1
+  AND s.hash IS NOT NULL
+  AND s.hash != ''
+  AND d.hash IS NULL
+GROUP BY s.hash, s.sourceId
+HAVING COUNT(*) > 1;
 `,
-/* ORDER BY [creation_time] ASC
-LIMIT $3 OFFSET ($2 - 1) * $3; */
       [sourceId, page, pageSize]
     );
     return Promise.resolve([true, res]);
@@ -516,4 +512,99 @@ LIMIT 1;`,
     }
     return Promise.resolve([false, error]);
   }
-}
+}
+
+/**
+ * 重复文件数据
+ * */
+export async function setDuplicateFile(sourceId: string, {
+  path,
+  type,
+  name,
+  hash,
+  creation_time,
+  modified_time,
+  file_size,
+  ids
+}: insertSearchFilesPasamsType) {
+  try {
+    const DB = await Database.load(`sqlite:files_${sourceId}.db`);
+    // 创建表
+    await DB.execute(createSql.duplicate_files);
+    await DB.execute(
+      `
+        INSERT into duplicate_files 
+          (create_time, sourceId, name, type, path, hash, creation_time, modified_time, file_size, db_version, ids) 
+        VALUES 
+          ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+      `,
+      [
+        new Date().getTime(),
+        sourceId,
+        name,
+        type,
+        path,
+        hash,
+        creation_time,
+        modified_time,
+        file_size,
+        "1",
+        ids
+      ]
+    );
+    return Promise.resolve([true, ""]);
+  } catch (error) {
+    if (error && `${error}`.indexOf("UNIQUE constraint failed") > -1) {
+      return "当前数据格式异常";
+    }
+    return Promise.resolve([false, error]);
+  }
+}
+/**
+ * 按照分页数据请求重复文件列表内容
+ * @param page 当前请求的页码，代表用户想要访问的数据页。
+ * @param pageSize 每页展示的记录数量，决定了每次查询返回的数据条数。
+ * @returns 返回一个对象，其中包含两个属性：
+ *          - data: FileInfoType[] - 当前页的记录数据数组。
+ *          - total: number - 表中的总记录数，用于前端计算总页数。
+ * */
+export async function getDuplicateFile(
+  {
+    page,
+    pageSize,
+    sourceId
+  }: 
+  {
+    page: number,
+    pageSize: number,
+    sourceId: string
+  }
+): Promise<{
+  [x: string]: any;
+  data: insertSearchFilesPasamsType[];
+  total: number;
+}>  {
+  try {
+    const DB = await Database.load(`sqlite:files_${sourceId}.db`);
+    // 创建表
+    await DB.execute(createSql.duplicate_files);
+    // 查询总记录数
+    const totalResult = await DB.select(
+      "SELECT COUNT(*) AS total FROM duplicate_files"
+    );
+    const total = Array.isArray(totalResult) && totalResult[0].total; // 获取总记录数
+    // 计算分页偏移量
+    const offset = (page || 1 - 1) * (pageSize || 10);
+    // 获取当前页的数据
+    const data = await DB.select(
+      "SELECT * FROM duplicate_files LIMIT ? OFFSET ?",
+      [pageSize, offset]
+    );
+    return { data: Array.isArray(data) ? data : [], total }; // 返回包含数据和总记录数的对象
+  } catch (error) {
+    return { data: [], total: 0 };
+  }
+}
+
+
+

src/types/files.d.ts

@@ -35,6 +35,7 @@ export type insertSearchFilesPasamsType = {
   file_size: string,
   creation_time?: string,
   modified_time?: string,
+  ids?: string,
 };
 
 export type historyListType = {

src/utils/commons.tsx

@@ -6,4 +6,18 @@ export function copyText(templateString: string) {
     document.execCommand('copy');
     document.body.removeChild(textArea);
     // message.success('账户复制成功!');
-  }
+  }
+
+ export function formatFileSize(bytes: number) {
+    if (bytes < 1024) {
+        return bytes + ' bytes';
+    } else if (bytes < 1024 * 1024) {
+        return (bytes / 1024).toFixed(2) + ' KB';
+    } else if (bytes < 1024 * 1024 * 1024) {
+        return (bytes / 1024 / 1024).toFixed(2) + ' MB';
+    } else if (bytes < 1024 * 1024 * 1024 * 1024) {
+        return (bytes / 1024 / 1024 / 1024).toFixed(2) + ' GB';
+    } else {
+        return (bytes / 1024 / 1024 / 1024 / 1024).toFixed(2) + ' TB';
+    }
+}

yarn.lock

@@ -881,11 +881,16 @@ csstype@^3.0.2, csstype@^3.1.3:
   resolved "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81"
   integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==
 
-dayjs@^1.11.10, dayjs@^1.11.11:
+dayjs@^1.11.10:
   version "1.11.11"
   resolved "https://registry.npmjs.org/dayjs/-/dayjs-1.11.11.tgz#dfe0e9d54c5f8b68ccf8ca5f72ac603e7e5ed59e"
   integrity sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg==
 
+dayjs@^1.11.11:
+  version "1.11.11"
+  resolved "https://registry.npmmirror.com/dayjs/-/dayjs-1.11.11.tgz#dfe0e9d54c5f8b68ccf8ca5f72ac603e7e5ed59e"
+  integrity sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg==
+
 debug@^4.1.0, debug@^4.3.1:
   version "4.3.5"
   resolved "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e"