Visitar URL original
perf(files): page the Files browser and resolve files without listing… · simstudioai/sim@70c8231 · GitHub
Skip to content

Commit 70c8231

Browse files
waleedlatif1waleedlatif1
authored
perf(files): page the Files browser and resolve files without listing the workspace (#9436)
Moves every workspace-file consumer off the full workspace list onto the paged, search and lookup reads: the Files browser is a virtualized infinite table with server-side sort (incl. Type and Owner), kind/size/uploader filters, ranked search and folder sizes; single-file views read the record by id; chips, names, image dimensions and stale-key recovery use batched lookups; pickers, mentions and Cmd+K use one recent-or-search hook; the sidebar flyout loads folders lazily. Type and kind rules are shared with the project file browser in JS and SQL. Writes patch only reads they can't reorder; the table separates its load-more spinner from the sentinel gate and waits for re-entry after a failed page. The unpaged list response and the public v1 API are unchanged.
1 parent fffa150 commit 70c8231

108 files changed

Lines changed: 5792 additions & 1456 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/sim/app/api/workspaces/[id]/files/folder-stats/route.test.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,22 @@ describe('GET /api/workspaces/[id]/files/folder-stats', () => {
3636
)
3737

3838
expect(response.status).toBe(200)
39-
expect(stats.mock.calls[0][0].input).toEqual({ workspaceId: WORKSPACE_ID, parentId: null })
39+
expect(stats.mock.calls[0][0].input).toEqual({ workspaceId: WORKSPACE_ID, parentIds: [null] })
40+
})
41+
42+
it('reads several parents at once, the root spelled `root`, in place of parentId', async () => {
43+
const response = await GET(
44+
createMockRequest({
45+
method: 'GET',
46+
url: `/api/workspaces/${WORKSPACE_ID}/files/folder-stats?parentId=folder-9&parentIds=folder-2,root`,
47+
}),
48+
createRouteContext({ id: WORKSPACE_ID })
49+
)
50+
51+
expect(response.status).toBe(200)
52+
expect(stats.mock.calls[0][0].input).toEqual({
53+
workspaceId: WORKSPACE_ID,
54+
parentIds: ['folder-2', null],
55+
})
4056
})
4157
})

‎apps/sim/app/api/workspaces/[id]/files/folder-stats/route.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,12 @@ export const GET = defineInternalJsonRoute({
2020
operation: fileOperations.list,
2121
rateLimit: internalRateLimits.user({ bucketName: 'workspace-files.read' }),
2222
errorPolicy: internalFileErrorPolicies.default,
23-
mapInput: ({ params, query }) => ({ workspaceId: params.id, parentId: query.parentId }),
23+
mapInput: ({ params, query }) => ({
24+
workspaceId: params.id,
25+
parentIds: query.parentIds
26+
? query.parentIds.map((parentId) => (parentId === 'root' ? null : parentId))
27+
: [query.parentId],
28+
}),
2429
useCase: getWorkspaceFileFolderStats,
2530
present: ({ folders }) => ({ success: true as const, folders }),
2631
})

‎apps/sim/app/api/workspaces/[id]/files/route.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,17 @@ describe('/api/workspaces/[id]/files', () => {
109109
})
110110
})
111111

112+
it('reads an empty set filter as no filter, keeping the whole-scope listing', async () => {
113+
collection.mockResolvedValueOnce({ files: [FILE], nextKeys: null })
114+
const response = await list('?scope=active&types=&sizes=&uploadedBy=')
115+
116+
expect(response.status).toBe(200)
117+
expect(collection.mock.calls[0][0].input).toEqual({
118+
workspaceId: WORKSPACE_ID,
119+
scope: 'active',
120+
})
121+
})
122+
112123
it('resumes from its own cursor and refuses it under another folder', async () => {
113124
collection.mockResolvedValue({ files: [FILE], nextKeys: ['notes.md', FILE.id] })
114125

@@ -127,5 +138,36 @@ describe('/api/workspaces/[id]/files', () => {
127138
expect(moved.status).toBe(400)
128139
expect(collection).toHaveBeenCalledTimes(2)
129140
})
141+
142+
it('binds its cursor to the search and set filters, whatever order the sets are spelled in', async () => {
143+
collection.mockResolvedValue({ files: [FILE], nextKeys: ['notes.md', FILE.id] })
144+
const filtered =
145+
'?sortBy=name&limit=1&q=notes&types=image,document&sizes=small&uploadedBy=u1,u2'
146+
147+
const first = await (await list(filtered)).json()
148+
expect(collection.mock.calls[0][0].input.page).toMatchObject({
149+
search: 'notes',
150+
kinds: ['document', 'image'],
151+
sizeBuckets: ['small'],
152+
uploadedBy: ['u1', 'u2'],
153+
})
154+
155+
const respelled = await list(
156+
`?sortBy=name&limit=1&q=notes&types=document,image,image&sizes=small&uploadedBy=u2,u1&cursor=${first.nextCursor}`
157+
)
158+
expect(respelled.status).toBe(200)
159+
160+
for (const changed of [
161+
'q=other&types=image,document&sizes=small&uploadedBy=u1,u2',
162+
'q=notes&types=image&sizes=small&uploadedBy=u1,u2',
163+
'q=notes&types=image,document&sizes=large&uploadedBy=u1,u2',
164+
'q=notes&types=image,document&sizes=small&uploadedBy=u1',
165+
'q=notes&types=image,document&sizes=small&uploadedBy=u1,u2&folderId=root&recursive=true',
166+
]) {
167+
const refiltered = await list(`?sortBy=name&limit=1&${changed}&cursor=${first.nextCursor}`)
168+
expect(refiltered.status).toBe(400)
169+
}
170+
expect(collection).toHaveBeenCalledTimes(2)
171+
})
130172
})
131173
})

‎apps/sim/app/api/workspaces/[id]/files/route.ts‎

Lines changed: 6 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
import {
22
createWorkspaceFileContract,
3-
DEFAULT_WORKSPACE_FILE_PAGE_SIZE,
43
listWorkspaceFilesContract,
5-
WORKSPACE_FILE_PAGE_PARAMS,
64
} from '@/lib/api/contracts/workspace-files'
7-
import { cursorRoute, cursorScopeKey } from '@/lib/api/cursor-binding'
85
import {
96
defineInternalJsonRoute,
107
internalRateLimits,
@@ -16,28 +13,20 @@ import {
1613
internalFileErrorPolicies,
1714
internalFilePresenters,
1815
} from '@/lib/workspace-files/api'
16+
import {
17+
presentWorkspaceFileList,
18+
toWorkspaceFileListInput,
19+
} from '@/lib/workspace-files/api/list-page'
1920
import {
2021
admitCreateWorkspaceFile,
2122
createWorkspaceFile,
2223
} from '@/lib/workspace-files/application/create-workspace-file'
2324
import { listWorkspaceFileCollection } from '@/lib/workspace-files/application/list-workspace-files'
2425
import { fileOperations } from '@/lib/workspace-files/application/operations'
2526
import { MAX_WORKSPACE_FILE_INLINE_BODY_BYTES } from '@/lib/workspace-files/orchestration'
26-
import { readSortedCursor, writeSortedCursor } from '@/app/api/v2/lib/response'
2727

2828
export const dynamic = 'force-dynamic'
2929

30-
/** The params besides the sort that change which files a page walk returns; the cursor is bound to them. */
31-
function fileCursorFilters(
32-
workspaceId: string,
33-
query: { scope: 'active' | 'archived'; folderId?: string | null }
34-
) {
35-
return cursorScopeKey(cursorRoute(listWorkspaceFilesContract, { id: workspaceId }), {
36-
scope: query.scope,
37-
folderId: query.folderId,
38-
})
39-
}
40-
4130
/**
4231
* GET /api/workspaces/[id]/files — List workspace files: one bounded page with `nextCursor`
4332
* when any paging param is sent, else the whole scope.
@@ -50,43 +39,9 @@ export const GET = defineInternalJsonRoute({
5039
// can choose a policy per request.
5140
rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal file-list behavior' }),
5241
errorPolicy: internalFileErrorPolicies.default,
53-
mapInput: ({ params, query }) => {
54-
if (!WORKSPACE_FILE_PAGE_PARAMS.some((param) => query[param] !== undefined)) {
55-
return { workspaceId: params.id, scope: query.scope }
56-
}
57-
const sortBy = query.sortBy ?? 'uploadedAt'
58-
const sortOrder = query.sortOrder ?? 'asc'
59-
return {
60-
workspaceId: params.id,
61-
scope: query.scope,
62-
page: {
63-
folderId: query.folderId,
64-
sortBy,
65-
sortOrder,
66-
limit: query.limit ?? DEFAULT_WORKSPACE_FILE_PAGE_SIZE,
67-
after: readSortedCursor(
68-
query.cursor,
69-
sortBy,
70-
sortOrder,
71-
fileCursorFilters(params.id, query)
72-
),
73-
},
74-
}
75-
},
42+
mapInput: ({ params, query }) => toWorkspaceFileListInput(params.id, query),
7643
useCase: listWorkspaceFileCollection,
77-
present: (result, { input }) => {
78-
const listed = internalFilePresenters.successFiles(result)
79-
if (!input.page) return listed
80-
return {
81-
...listed,
82-
nextCursor: writeSortedCursor(
83-
result.nextKeys,
84-
input.page.sortBy,
85-
input.page.sortOrder,
86-
fileCursorFilters(input.workspaceId, { scope: input.scope, folderId: input.page.folderId })
87-
),
88-
}
89-
},
44+
present: (result, { input }) => presentWorkspaceFileList(result, input),
9045
})
9146

9247
/** POST /api/workspaces/[id]/files — Create an authored workspace file. */

‎apps/sim/app/api/workspaces/[id]/files/search/route.ts‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import {
55
internalSessionAuth,
66
} from '@/lib/api/server/routes'
77
import { internalFileErrorPolicies, internalFilePresenters } from '@/lib/workspace-files/api'
8-
import { listWorkspaceFileCollection } from '@/lib/workspace-files/application/list-workspace-files'
8+
import { searchWorkspaceFiles } from '@/lib/workspace-files/application/list-workspace-files'
99
import { fileOperations } from '@/lib/workspace-files/application/operations'
1010

1111
export const dynamic = 'force-dynamic'
@@ -14,8 +14,9 @@ export const dynamic = 'force-dynamic'
1414
const SEARCH_TYPE_AHEAD_LIMIT = { maxTokens: 120, refillRate: 120, refillIntervalMs: 60_000 }
1515

1616
/**
17-
* GET /api/workspaces/[id]/files/search — live files whose name contains `q`, by name, each
18-
* carrying its `folderPath`; within `folderId` (and, by default, its subfolders) when given.
17+
* GET /api/workspaces/[id]/files/search — live files `q` finds, best match first: by name
18+
* (exact, prefix, substring, then fuzzy), then by the folders they sit under; each carries its
19+
* `folderPath`. Within `folderId` (and, by default, its subfolders) when given.
1920
*/
2021
export const GET = defineInternalJsonRoute({
2122
contract: searchWorkspaceFilesContract,
@@ -28,16 +29,11 @@ export const GET = defineInternalJsonRoute({
2829
errorPolicy: internalFileErrorPolicies.default,
2930
mapInput: ({ params, query }) => ({
3031
workspaceId: params.id,
31-
scope: 'active' as const,
32-
page: {
33-
search: query.q,
34-
folderId: query.folderId,
35-
recursive: query.recursive,
36-
sortBy: 'name' as const,
37-
sortOrder: 'asc' as const,
38-
limit: query.limit,
39-
},
32+
query: query.q,
33+
folderId: query.folderId,
34+
recursive: query.recursive,
35+
limit: query.limit,
4036
}),
41-
useCase: listWorkspaceFileCollection,
37+
useCase: searchWorkspaceFiles,
4238
present: internalFilePresenters.successFiles,
4339
})

‎apps/sim/app/o/[organizationId]/home/organization-home.tsx‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ import {
3636
useResourcePanelController,
3737
} from '@/app/workspace/[workspaceId]/home/hooks/use-resource-panel'
3838
import { useRestoredChatEntry } from '@/app/workspace/[workspaceId]/home/hooks/use-restored-chat-entry'
39-
import { resolveWorkspaceResourceRef } from '@/app/workspace/[workspaceId]/home/resolve-resource-ref'
39+
import { fetchWorkspaceResourceRef } from '@/app/workspace/[workspaceId]/home/resolve-resource-ref'
4040
import { searchFiltersFromParams } from '@/app/workspace/[workspaceId]/home/search-params'
4141
import type {
4242
ChatRequestMode,
@@ -47,7 +47,6 @@ import { useFeatureFlag } from '@/app/workspace/[workspaceId]/providers/feature-
4747
import { useFileAttachments } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments'
4848
import { mentionifyIntegrations } from '@/blocks/integration-matcher'
4949
import { useMarkMothershipChatRead } from '@/hooks/queries/mothership-chats'
50-
import { getWorkspaceFilesQueryOptions } from '@/hooks/queries/workspace-files'
5150
import { useHydrated } from '@/hooks/use-hydrated'
5251
import { useMothershipDraftsStore } from '@/stores/mothership-drafts/store'
5352
import { useOrganizationChatModeStore } from '@/stores/organization-chat-mode/store'
@@ -180,11 +179,7 @@ function OrganizationHomeContent({
180179
toast.error('This resource is outside this organization.')
181180
return
182181
}
183-
const files =
184-
ref.type === 'file'
185-
? await queryClient.fetchQuery(getWorkspaceFilesQueryOptions(ref.workspaceId))
186-
: []
187-
const resource = resolveWorkspaceResourceRef(ref, files)
182+
const resource = await fetchWorkspaceResourceRef(queryClient, ref.workspaceId, ref)
188183
if (!resource) {
189184
toast.error(`Couldn't find "${ref.title}" in its workspace`)
190185
return

‎apps/sim/app/workspace/[workspaceId]/changelog/components/release-entry/release-entry.tsx‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import Link from 'next/link'
1717
import type { ChangelogRelease } from '@/lib/api/contracts/changelog'
1818
import { ReleaseWorkflows } from '@/app/workspace/[workspaceId]/changelog/components/release-workflows/release-workflows'
1919
import { FileViewer } from '@/app/workspace/[workspaceId]/files/components/file-viewer'
20-
import { useAddressedWorkspaceFileRecord } from '@/hooks/queries/workspace-files'
20+
import { useWorkspaceFileRecord } from '@/hooks/queries/workspace-files'
2121

2222
interface ReleaseEntryProps {
2323
workspaceId: string
@@ -34,7 +34,7 @@ export function ReleaseEntry({ workspaceId, release, canEdit, onEditDetails }: R
3434
const saveFailedRef = useRef(false)
3535
const [editingBody, setEditingBody] = useState(false)
3636
const [savingBody, setSavingBody] = useState(false)
37-
const file = useAddressedWorkspaceFileRecord(workspaceId, release.fileId)
37+
const file = useWorkspaceFileRecord(workspaceId, release.fileId)
3838

3939
return (
4040
<article className='grid @min-[760px]/changelog:grid-cols-[132px_minmax(0,1fr)] grid-cols-1 gap-x-10 gap-y-3 border-[var(--border)] border-t pt-12 first:border-t-0 first:pt-0'>
@@ -95,9 +95,9 @@ export function ReleaseEntry({ workspaceId, release, canEdit, onEditDetails }: R
9595
))}
9696
</div>
9797

98-
{file.error ? (
98+
{file.error || file.data === null ? (
9999
<p role='alert' className='text-[var(--text-error)] text-small'>
100-
{file.error.message}
100+
{file.error?.message ?? 'File not found'}
101101
</p>
102102
) : file.data ? (
103103
<div

‎apps/sim/app/workspace/[workspaceId]/chat/[chatId]/page.tsx‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
11
import { Suspense } from 'react'
2-
import { dehydrate, HydrationBoundary } from '@tanstack/react-query'
32
import type { Metadata } from 'next'
43
import { notFound } from 'next/navigation'
54
import { getSession } from '@/lib/auth'
65
import { isChatEnabled } from '@/lib/core/config/env-flags'
7-
import { getQueryClient } from '@/app/_shell/providers/get-query-client'
86
import { Home } from '@/app/workspace/[workspaceId]/home/home'
97
import { HomeFallback } from '@/app/workspace/[workspaceId]/home/home-fallback'
10-
import { prefetchHomeSurface } from '@/app/workspace/[workspaceId]/home/prefetch'
118

129
export const metadata: Metadata = {
1310
title: 'Chat',
@@ -22,20 +19,20 @@ interface ChatPageProps {
2219

2320
export default async function ChatPage({ params }: ChatPageProps) {
2421
// The layout 404s too, but pages and layouts resolve concurrently — without this
25-
// the prefetch below still fires on its way out.
22+
// the page still renders on its way out.
2623
if (!isChatEnabled) {
2724
notFound()
2825
}
2926

30-
const [{ workspaceId, chatId }, session] = await Promise.all([params, getSession()])
31-
const userId = session?.user?.id
32-
const queryClient = getQueryClient()
33-
await prefetchHomeSurface(queryClient, workspaceId, userId)
27+
const [{ chatId }, session] = await Promise.all([params, getSession()])
3428
return (
35-
<HydrationBoundary state={dehydrate(queryClient)}>
36-
<Suspense fallback={<HomeFallback />}>
37-
<Home key={chatId} chatId={chatId} userName={session?.user?.name} userId={userId} />
38-
</Suspense>
39-
</HydrationBoundary>
29+
<Suspense fallback={<HomeFallback />}>
30+
<Home
31+
key={chatId}
32+
chatId={chatId}
33+
userName={session?.user?.name}
34+
userId={session?.user?.id}
35+
/>
36+
</Suspense>
4037
)
4138
}

‎apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.test.ts‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { describe, expect, it } from 'vitest'
2-
import { buildFlyoutEntries } from '@/app/workspace/[workspaceId]/components/folders/flyout-entries'
2+
import {
3+
buildFlyoutEntries,
4+
flyoutLevelFolders,
5+
} from '@/app/workspace/[workspaceId]/components/folders/flyout-entries'
36

47
function folder(id: string, name: string, parentId: string | null, updatedAt: string) {
58
return { id, name, parentId, updatedAt: new Date(updatedAt) }
@@ -49,3 +52,21 @@ describe('buildFlyoutEntries', () => {
4952
expect(entries.map((entry) => entry.id)).toEqual(['root'])
5053
})
5154
})
55+
56+
describe('flyoutLevelFolders', () => {
57+
const folders = [
58+
folder('docs', 'Docs', null, '2026-01-01'),
59+
folder('sub', 'Sub', 'docs', '2026-01-01'),
60+
folder('orphan', 'Orphan', 'archived-parent', '2026-01-01'),
61+
]
62+
63+
it('lists the root children and the folders whose parent is gone at the root', () => {
64+
expect(flyoutLevelFolders(folders, null).map((entry) => entry.id)).toEqual(['docs', 'orphan'])
65+
})
66+
67+
it("lists only a folder's own children one level down", () => {
68+
expect(flyoutLevelFolders(folders, 'docs').map((entry) => [entry.id, entry.parentId])).toEqual([
69+
['sub', null],
70+
])
71+
})
72+
})

‎apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,25 @@ export interface BuildFlyoutEntriesParams<Item extends FlyoutItemSource> {
3232
hrefForItem: (item: Item) => string
3333
}
3434

35+
/**
36+
* The folders one lazily built flyout level lists, as root folders of that level: `parentId`'s
37+
* children, and at the root (`null`) also every folder whose parent is not among `folders` — an
38+
* archived or missing parent — which {@link buildFlyoutEntries} surfaces at the root of a
39+
* whole tree, so its contents stay reachable one level at a time too.
40+
*/
41+
export function flyoutLevelFolders<Folder extends FlyoutFolderSource>(
42+
folders: readonly Folder[],
43+
parentId: string | null
44+
): Folder[] {
45+
const folderIds = new Set(folders.map((folder) => folder.id))
46+
return folders
47+
.filter((folder) => {
48+
const parent = folder.parentId ?? null
49+
return parentId === null ? parent === null || !folderIds.has(parent) : parent === parentId
50+
})
51+
.map((folder) => ({ ...folder, parentId: null }))
52+
}
53+
3554
function flyoutSortTime(value: Date | string): number {
3655
const time = value instanceof Date ? value.getTime() : Date.parse(value)
3756
return Number.isNaN(time) ? 0 : time

0 commit comments

Comments
 (0)