diff --git a/CHANGELOG.md b/CHANGELOG.md index 45f4d08c7..538eab24e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Made the default home page configurable with `DEFAULT_HOME_VIEW_PAGE`, defaulting to Code Search and supporting Ask. [#1677](https://github.com/sourcebot-dev/sourcebot/pull/1677) - Require authentication for the streaming and blocking Ask APIs in Public SaaS deployments. [#1679](https://github.com/sourcebot-dev/sourcebot/pull/1679) +- Fixed local repositories whose path contains spaces or non-ASCII characters resolving to a percent-encoded path that could not be indexed. [#1690](https://github.com/sourcebot-dev/sourcebot/pull/1690) ## [5.1.14] - 2026-09-17 diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index cdc78d8e0..f5198c6b9 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -14,6 +14,7 @@ import { createLogger } from '@sourcebot/shared'; import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js"; import path from 'path'; +import { pathToFileURL } from 'url'; import fs from 'fs/promises'; import { glob } from 'glob'; import { getLocalDefaultBranch, getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js'; @@ -658,7 +659,7 @@ export const compileGenericGitHostConfig_file = async ( external_codeHostType: 'genericGitHost', external_codeHostUrl: remoteUrl.resource, external_id: remoteUrl.toString(), - cloneUrl: `file://${repoPath}`, + cloneUrl: pathToFileURL(repoPath).href, name: repoName, displayName: repoName, defaultBranch, diff --git a/packages/shared/src/utils.test.ts b/packages/shared/src/utils.test.ts index 14e1de4e8..22503adc6 100644 --- a/packages/shared/src/utils.test.ts +++ b/packages/shared/src/utils.test.ts @@ -1,7 +1,17 @@ import { readFile } from 'fs/promises'; +import { existsSync, mkdtempSync, mkdirSync, rmSync } from 'fs'; +import { tmpdir } from 'os'; +import path from 'path'; +import { fileURLToPath, pathToFileURL } from 'url'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import { DEFAULT_CONFIG_SETTINGS } from './constants.js'; -import { getConfigSettings, resolveConfigSettings } from './utils.js'; +import type { Repo } from '@sourcebot/db'; +import { + getConfigSettings, + getRepoPath, + normalizeLegacyFileURLPathname, + resolveConfigSettings, +} from './utils.js'; // Mock fs/promises so loadConfig doesn't hit the filesystem. // The config schema has no required fields, so '{}' is valid. @@ -110,3 +120,83 @@ describe('resolveConfigSettings', () => { ); }); }); + +describe('getRepoPath', () => { + const localRepo = (cloneUrl: string) => ({ + id: 1, + external_codeHostType: 'genericGitHost', + cloneUrl, + }) as unknown as Repo; + + const localPath = (...parts: string[]) => + `${process.platform === 'win32' ? 'C:\\' : '/'}${parts.join('/')}`; + + test( + 'normalizes legacy Windows file URL pathnames before checking the filesystem', + () => { + const cloneUrl = new URL('file://C:\\Users\\me\\100%20Free'); + const nativePath = 'C:\\Users\\me\\100%20Free'; + const legacyPathname = cloneUrl.pathname; + + expect(legacyPathname).toBe('/C:/Users/me/100%20Free'); + expect(path.win32.normalize(legacyPathname)).not.toBe(nativePath); + expect( + normalizeLegacyFileURLPathname(legacyPathname, 'win32'), + ).toBe(nativePath); + expect( + normalizeLegacyFileURLPathname('/repos/100%20Free', 'win32'), + ).toBe('/repos/100%20Free'); + }, + ); + + test('returns the on-disk path of a local repository', () => { + const repoPath = localPath('repos', 'project'); + expect(getRepoPath(localRepo(pathToFileURL(repoPath).href))).toEqual({ + path: fileURLToPath(pathToFileURL(repoPath)), + isReadOnly: true, + }); + }); + + test('returns the on-disk path of a local repository whose path contains spaces', () => { + const repoPath = localPath('Users', 'me', 'Code Projects', 'my repo'); + expect(getRepoPath(localRepo(pathToFileURL(repoPath).href))).toEqual({ + path: fileURLToPath(pathToFileURL(repoPath)), + isReadOnly: true, + }); + }); + + test('returns the on-disk path of a local repository whose path contains percent-encodable characters', () => { + const repoPath = localPath('repos', 'caf\u00e9', '[legacy]'); + expect(getRepoPath(localRepo(pathToFileURL(repoPath).href))).toEqual({ + path: fileURLToPath(pathToFileURL(repoPath)), + isReadOnly: true, + }); + }); + + test.each(['file:///repos/100%Free', 'file:///repos/report%2F2024'])( + 'preserves a legacy raw file URL pathname containing %s', (cloneUrl) => { + const url = new URL(cloneUrl); + expect(getRepoPath(localRepo(cloneUrl))).toEqual({ + path: url.pathname, + isReadOnly: true, + }); + }, + ); + + test('preserves a legacy local path containing a valid percent escape when that path exists', () => { + const tempRoot = mkdtempSync(path.join(tmpdir(), 'sourcebot-local-repo-')); + const repoPath = path.join(tempRoot, '100%20Free'); + mkdirSync(repoPath); + + try { + // Older records stored `file://` plus the raw path, so `%20` here + // is a literal part of the directory name rather than an escape. + expect(getRepoPath(localRepo(`file://${repoPath}`))).toEqual({ + path: repoPath, + isReadOnly: true, + }); + } finally { + rmSync(tempRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/shared/src/utils.ts b/packages/shared/src/utils.ts index 2a1867fc4..5984d5967 100644 --- a/packages/shared/src/utils.ts +++ b/packages/shared/src/utils.ts @@ -1,4 +1,5 @@ import { readFile } from 'fs/promises'; +import { existsSync } from 'fs'; import stripJsonComments from 'strip-json-comments'; import { z } from "zod"; import { DEFAULT_CONFIG_SETTINGS } from "./constants.js"; @@ -6,6 +7,7 @@ import { ConfigSettings } from "./types.js"; import { Org, Repo } from "@sourcebot/db"; import type { SourcebotConfig } from "@sourcebot/schemas/v3/index.type"; import path from "path"; +import { fileURLToPath } from "url"; import { env, isRemotePath, loadConfig } from "./env.server.js"; // From https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem @@ -103,13 +105,40 @@ export const getRepoIdFromPath = (repoPath: string): number | undefined => { return isNaN(id) ? undefined : id; } +export const normalizeLegacyFileURLPathname = ( + pathname: string, + platform = process.platform, +): string => { + if (platform === 'win32' && /^\/[A-Za-z]:\//.test(pathname)) { + return path.win32.normalize(pathname.slice(1)); + } + return pathname; +} + export const getRepoPath = (repo: Repo): { path: string, isReadOnly: boolean } => { // If we are dealing with a local repository, then use that as the path. // Mark as read-only since we aren't guaranteed to have write access to the local filesystem. const cloneUrl = new URL(repo.cloneUrl); if (repo.external_codeHostType === 'genericGitHost' && cloneUrl.protocol === 'file:') { + let localPath: string; + try { + localPath = fileURLToPath(cloneUrl); + // Older records were written as `file://` plus the raw filesystem + // path. If a literal-percent path exists but its decoded spelling + // does not, keep the original path instead of treating `%20` as a + // space (or another percent escape as URL syntax). + const legacyPath = normalizeLegacyFileURLPathname(cloneUrl.pathname); + if (!existsSync(localPath) && existsSync(legacyPath)) { + localPath = legacyPath; + } + } catch { + // Older records may contain raw paths with invalid escapes such as + // `%2F`; retain the previous pathname behavior for those records. + localPath = normalizeLegacyFileURLPathname(cloneUrl.pathname); + } return { - path: cloneUrl.pathname, + // New clone URLs are filesystem-encoded; decode them to the path. + path: localPath, isReadOnly: true, } }