Skip to content

fix: Deduplicate chrome screenshots #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aklinker1 merged 1 commit into from
Oct 1, 2026
Merged

fix: Deduplicate chrome screenshots #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 0 additions & 130 deletions src/services/__tests__/__snapshots__/chrome-crawler.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,36 +20,6 @@ AI Integration: Extract and convert relevant website information into markdown f
"rating": 0,
"reviewCount": 0,
"screenshots": [
{
"index": 5,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/5",
"rawUrl": "https://lh3.googleusercontent.com/SwOe0XmN3ZkCkJQ4IF1a1i2qqCsw0b7I-Ez6VSQflAvafwiJhGbdRW6YAKNX8fAwdv159XogYSFc5Uykgt1fsYSv=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/1",
"rawUrl": "https://lh3.googleusercontent.com/vDnkawI_rgAd5ou0gasjR5CQbm-HmbTj9SBTdlpxU_-ri1jeE9dz0qNnSOijaboiUd4tuwQBGg-ujIO-znDN4-InVZA=s1280",
},
{
"index": 2,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/2",
"rawUrl": "https://lh3.googleusercontent.com/F9H2EBxfqxrriMb0zdjoFIiCufav1oV8QI-enLqT9AEuL0nPFJLaj0286R2UH_ekxLoVa-yO76f5sSDOAjEuORrRLg=s1280",
},
{
"index": 3,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/3",
"rawUrl": "https://lh3.googleusercontent.com/kkK_WlsYeZNGK-sU-4nvH-sEXtx7xUMuqpsK-IHq29IUkmyC3C40n1DDsDdt0f6-LgbKGL_obH_Cse-zJWJPHk-OLQ=s1280",
},
{
"index": 4,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/4",
"rawUrl": "https://lh3.googleusercontent.com/VJy9hYOMvlmwp2FhtrpIm8hNwwJVZWl-V4dZVl2cCl4aD8xNyuHaEgdm5bfjMBiZJXI0ysxul8P1GAyFaJrRvjDSuMY=s1280",
},
{
"index": 5,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/5",
"rawUrl": "https://lh3.googleusercontent.com/SwOe0XmN3ZkCkJQ4IF1a1i2qqCsw0b7I-Ez6VSQflAvafwiJhGbdRW6YAKNX8fAwdv159XogYSFc5Uykgt1fsYSv=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/kofbbilhmnkcmibjbioafflgmpkbnmme/screenshots/1",
Expand Down Expand Up @@ -160,36 +130,6 @@ Absolutely! Blync is 100% secure.
"rating": 5,
"reviewCount": 2,
"screenshots": [
{
"index": 5,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/5",
"rawUrl": "https://lh3.googleusercontent.com/MXf271aHeP9deq2VjO0-_QDU-VSKgu81hcc9iKNMbAg74sI1RnvOb7Esgy95P-NTrQ7vVJtW2fMOPnTYv8pnbnMEng=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/1",
"rawUrl": "https://lh3.googleusercontent.com/WVvtpr2Wzfr4zT34Z_Gy39ZL-I8UOZ5uReOAz-vmtC8lT4hebXlnur7y0bAWzXd6xihMbTP_PfZgQN_C4MfhixI2=s1280",
},
{
"index": 2,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/2",
"rawUrl": "https://lh3.googleusercontent.com/e3JEIuiGu_dI70zGBPBXzL9YNZuHy8nItCoEKJSql56CcUVyIQC9Lor8HkeScWE2-qAFw5DLZQijXEfNxlInSoLl0DE=s1280",
},
{
"index": 3,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/3",
"rawUrl": "https://lh3.googleusercontent.com/O9OuxL28A3GOp5-R8u2kyV9wkOchMTx0ZNWraAz93gqThZRwVpCbaEW0VL1bZ0dKaKr0ReN0qJYJfSFAPOkCuOGA=s1280",
},
{
"index": 4,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/4",
"rawUrl": "https://lh3.googleusercontent.com/vfXSijB8FQ4hkARrKdJJfXe3b_ahd8yXsSuAbbKNwU9yondg0SIj1fenPJRosimBMo8bi1hj99lypD1CS1y1KhJE=s1280",
},
{
"index": 5,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/5",
"rawUrl": "https://lh3.googleusercontent.com/MXf271aHeP9deq2VjO0-_QDU-VSKgu81hcc9iKNMbAg74sI1RnvOb7Esgy95P-NTrQ7vVJtW2fMOPnTYv8pnbnMEng=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/odffpjnpocjfcaclnenaaaddghkgijdb/screenshots/1",
Expand Down Expand Up @@ -251,51 +191,6 @@ With just two clicks, Eye Dropper allows you to pick colors from any website wit
"rating": 4.3,
"reviewCount": 3,
"screenshots": [
{
"index": 3,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/3",
"rawUrl": "https://lh3.googleusercontent.com/0YMjrWbn7_iSWpjvF45C2xdswzPDZZ8Qf2cPeM2-VrQF8PNX9F0sWWjr-CBbCOH0kogD0CEW2uewbx5TyvLAf1Ia=s1280",
},
{
"index": 4,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/4",
"rawUrl": "https://lh3.googleusercontent.com/YaXuIdUkKlr7g5tq2mpY8uWvckzknb6xyv9TD6iam0-6ri-7_a1jjntmTv3dz0x0kz38lEUEHW-r9q6zp0eMwPov=s1280",
},
{
"index": 0,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/0",
"rawUrl": "https://lh3.googleusercontent.com/cLm5vTn9bPNn6QvxuJdcqgP3JF47XB4hKm9rP3XibkyT2arxWo_HwCa6c8RwcIV4Et0c1TXfnDie4Xgj24svzbye=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/1",
"rawUrl": "https://lh3.googleusercontent.com/AeVZqwDPLzUMZb3JAV2qsD8KbxRnioA176cE3DYBANpYY3TyCQljLcfTKmlGvV5Xdt52AJkjT65ok8t4d0cYnobvIQ=s1280",
},
{
"index": 2,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/2",
"rawUrl": "https://lh3.googleusercontent.com/Hqn2xlSKEuxQfBe-YHPlhUMjox2G0ZpYd1ouFx9ATfb_YbwtqfVkp9v8AjBI9RzBghbH9yci-cxdoJnwL8VCskRZMQ=s1280",
},
{
"index": 3,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/3",
"rawUrl": "https://lh3.googleusercontent.com/0YMjrWbn7_iSWpjvF45C2xdswzPDZZ8Qf2cPeM2-VrQF8PNX9F0sWWjr-CBbCOH0kogD0CEW2uewbx5TyvLAf1Ia=s1280",
},
{
"index": 4,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/4",
"rawUrl": "https://lh3.googleusercontent.com/YaXuIdUkKlr7g5tq2mpY8uWvckzknb6xyv9TD6iam0-6ri-7_a1jjntmTv3dz0x0kz38lEUEHW-r9q6zp0eMwPov=s1280",
},
{
"index": 0,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/0",
"rawUrl": "https://lh3.googleusercontent.com/cLm5vTn9bPNn6QvxuJdcqgP3JF47XB4hKm9rP3XibkyT2arxWo_HwCa6c8RwcIV4Et0c1TXfnDie4Xgj24svzbye=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/1",
"rawUrl": "https://lh3.googleusercontent.com/AeVZqwDPLzUMZb3JAV2qsD8KbxRnioA176cE3DYBANpYY3TyCQljLcfTKmlGvV5Xdt52AJkjT65ok8t4d0cYnobvIQ=s1280",
},
{
"index": 0,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/hmdcmlfkchdmnmnmheododdhjedfccka/screenshots/0",
Expand Down Expand Up @@ -359,31 +254,6 @@ Upgrade your browsing experience—make every new tab a beautiful, functional wo
"rating": 4.6,
"reviewCount": 10,
"screenshots": [
{
"index": 2,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/2",
"rawUrl": "https://lh3.googleusercontent.com/gj2e-tiTc3QYptxlIEcVfRogxZcXaJ_uQu09lK4wwI9ih1f765cnB4Vx-Z2m3ib7YxxeWWonL5ZF2CAHZCAcVck6=s1280",
},
{
"index": 0,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/0",
"rawUrl": "https://lh3.googleusercontent.com/5cf2eynPB11FLePhVBJmft3XXcJJ9Y6ZPL_jcPB2T-eX1k2YW5JORQN5kPvPzvQfQH2GytQWi7iTB1sEfSkafoLY7Q=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/1",
"rawUrl": "https://lh3.googleusercontent.com/fZfIjaptorXfgT-JAsfUsqpHsxweFuMmqfDafphvbDsGdaKtLROsUPeE8_6kmI0QIHOotPMoKiEWbIpcB1wajTaGx_o=s1280",
},
{
"index": 2,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/2",
"rawUrl": "https://lh3.googleusercontent.com/gj2e-tiTc3QYptxlIEcVfRogxZcXaJ_uQu09lK4wwI9ih1f765cnB4Vx-Z2m3ib7YxxeWWonL5ZF2CAHZCAcVck6=s1280",
},
{
"index": 0,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/0",
"rawUrl": "https://lh3.googleusercontent.com/5cf2eynPB11FLePhVBJmft3XXcJJ9Y6ZPL_jcPB2T-eX1k2YW5JORQN5kPvPzvQfQH2GytQWi7iTB1sEfSkafoLY7Q=s1280",
},
{
"index": 0,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/dhiekgdaipindoapjmcnpompdknjeijf/screenshots/0",
Expand Down
12 changes: 5 additions & 7 deletions src/services/__tests__/chrome-crawler.e2e.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,11 @@ describe("Chrome Web Store Crawler E2E", () => {
version: expect.any(String),
users: expect.any(Number),
weeklyActiveUsers: expect.any(Number),
screenshots: expect.arrayContaining(
[0, 1, 2, 3, 4].map((index) => ({
index,
indexUrl: `http://localhost:3000/api/rest/chrome-web-store/${animeSkipPlayerId}/screenshots/${index}`,
rawUrl: expect.any(String),
})),
),
screenshots: [0, 1, 2, 3, 4].map((index) => ({
index,
indexUrl: `http://localhost:3000/api/rest/chrome-web-store/${animeSkipPlayerId}/screenshots/${index}`,
rawUrl: expect.any(String),
})),
});
});
});
34 changes: 23 additions & 11 deletions src/services/chrome-crawler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,18 +126,30 @@ export async function crawlExtension(
?.textContent?.replaceAll("\n\n", "\n"),
]);

const extractScreenshots = (includeCarousel: boolean): Gql.Screenshot[] => {
const screenshots = [...document.querySelectorAll("div[data-media-url]")]
.filter((div) => div.getAttribute("data-is-video") === "false")
// The main carousel contains cloned slides at both ends for infinite scrolling, so by
// default only use the preview thumbnails, which are listed exactly once.
.filter((div) => includeCarousel || !div.closest('[role="region"]'))
.map<Gql.Screenshot>((div) => {
const index = parseInt(div.getAttribute("data-slide-index"));
return {
index,
rawUrl: div.getAttribute("data-media-url") + "=s1280", // "s1280" gets the full resolution
indexUrl: buildScreenshotUrl(ExtensionStoreName.ChromeWebStore, id, index),
};
});
if (!includeCarousel) return screenshots;

// Remove duplicates from the carousel's cloned slides
return [...new Map(screenshots.map((s) => [s.index, s])).values()].sort(
(a, b) => a.index - b.index,
);
};
const screenshots = tryExtract("screenshots", validateGqlScreenshots, [
() =>
[...document.querySelectorAll("div[data-media-url]")]
.filter((div) => div.getAttribute("data-is-video") === "false")
.map<Gql.Screenshot>((div) => {
const index = parseInt(div.getAttribute("data-slide-index"));
return {
index,
rawUrl: div.getAttribute("data-media-url") + "=s1280", // "s1280" gets the full resolution
indexUrl: buildScreenshotUrl(ExtensionStoreName.ChromeWebStore, id, index),
};
}),
() => extractScreenshots(false),
() => extractScreenshots(true),
]);

const result: Gql.ChromeExtension = {
Expand Down
Loading