update search filter code (#4317)

* sync collection * remove lock * update search filter code

update search filter code (#4317)
* sync collection * remove lock * update search filter code
ff64a3c0 · Archer · archer · 37b4a191 · ff64a3c0
Commit ff64a3c0 authored Mar 25, 2025 by Archer Committed by archer Mar 27, 2025
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 4 deletions

packages/service/core/dataset/search/controller.ts
+2 -4

No files found.
--- a/packages/service/core/dataset/search/controller.ts
+++ b/packages/service/core/dataset/search/controller.ts
@@ -134,12 +134,10 @@ export const filterDatasetDataByMaxTokens = async (
    let totalTokens = 0;

    for await (const item of tokensScoreFilter) {
+      results.push(item);
+
      totalTokens += item.tokens;

-      if (totalTokens > maxTokens + 500) {
-        break;
-      }
-      results.push(item);
      if (totalTokens > maxTokens) {
        break;
      }