From 6d6a2d2a11f2d3c4d2c2a536363f7aa80ee97359 Mon Sep 17 00:00:00 2001 From: kawamataryo Date: Fri, 22 Nov 2024 21:33:21 +0900 Subject: [PATCH] feat: support threds --- package.json | 3 +- src/contents/App.tsx | 19 +++++- src/lib/constants.ts | 10 +++ src/lib/hooks/useRetrieveBskyUsers.ts | 88 +++++++++++++++++++-------- src/lib/services/abstractService.ts | 29 +++++---- src/lib/services/threadsService.ts | 80 ++++++++++++++++++++++++ src/lib/services/xService.ts | 9 +++ src/lib/utils.ts | 25 ++++++++ src/popup.tsx | 17 +++++- src/types.ts | 8 ++- 10 files changed, 244 insertions(+), 44 deletions(-) create mode 100644 src/lib/services/threadsService.ts diff --git a/package.json b/package.json index f5ca96f..b7555f6 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,8 @@ "host_permissions": [ "https://bsky.social/*", "https://twitter.com/*", - "https://x.com/*" + "https://x.com/*", + "https://www.threads.net/*" ], "browser_specific_settings": { "gecko": { diff --git a/src/contents/App.tsx b/src/contents/App.tsx index e20f60a..247740a 100644 --- a/src/contents/App.tsx +++ b/src/contents/App.tsx @@ -4,14 +4,19 @@ import { Storage } from "@plasmohq/storage"; import { useStorage } from "@plasmohq/storage/hook"; import type { PlasmoCSConfig } from "plasmo"; import React from "react"; +import { match } from "ts-pattern"; import AlertError from "~lib/components/AlertError"; import LoadingCards from "~lib/components/LoadingCards"; import Modal from "~lib/components/Modal"; -import { MESSAGE_NAMES, STORAGE_KEYS } from "~lib/constants"; +import { MESSAGE_NAMES, SERVICE_TYPE, STORAGE_KEYS } from "~lib/constants"; import { useRetrieveBskyUsers } from "~lib/hooks/useRetrieveBskyUsers"; export const config: PlasmoCSConfig = { - matches: ["https://twitter.com/*", "https://x.com/*"], + matches: [ + "https://twitter.com/*", + "https://x.com/*", + "https://www.threads.net/*", + ], all_frames: true, }; @@ -31,6 +36,7 @@ const App = () => { restart, isBottomReached, errorMessage, + currentService, } = useRetrieveBskyUsers(); const [isModalOpen, setIsModalOpen] = React.useState(false); @@ -89,13 +95,20 @@ const App = () => { stopRetrieveLoop(); }; + const serviceName = React.useMemo(() => { + return match(currentService) + .with(SERVICE_TYPE.X, () => "X") + .with(SERVICE_TYPE.THREADS, () => "Threads") + .exhaustive(); + }, [currentService]); + return ( <>
{loading && (

- Scanning 𝕏 users to find bsky users... + Scanning {serviceName} users to find bsky users...

)}

diff --git a/src/lib/constants.ts b/src/lib/constants.ts index cbcd323..083ff8d 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -3,18 +3,21 @@ export const MESSAGE_NAMES = { SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE: "search_bsky_user_on_list_members_page", SEARCH_BSKY_USER_ON_BLOCK_PAGE: "search_bsky_user_on_block_page", + SEARCH_BSKY_USER_ON_THREADS_PAGE: "search_bsky_user_on_threads_page", } as const; export const QUERY_PARAMS = { FOLLOW: '[data-testid="primaryColumn"] [data-testid="UserCell"]', BLOCK: '[data-testid="UserCell"]', LIST: '[data-testid="cellInnerDiv"] [data-testid="UserCell"]', + THREADS: '[data-pressable-container="true"]', } as const; export const MESSAGE_NAME_TO_QUERY_PARAM_MAP = { [MESSAGE_NAMES.SEARCH_BSKY_USER_ON_FOLLOW_PAGE]: QUERY_PARAMS.FOLLOW, [MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE]: QUERY_PARAMS.LIST, [MESSAGE_NAMES.SEARCH_BSKY_USER_ON_BLOCK_PAGE]: QUERY_PARAMS.BLOCK, + [MESSAGE_NAMES.SEARCH_BSKY_USER_ON_THREADS_PAGE]: QUERY_PARAMS.THREADS, }; export const ACTION_MODE = { @@ -28,6 +31,7 @@ export const MESSAGE_NAME_TO_ACTION_MODE_MAP = { [MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE]: ACTION_MODE.IMPORT_LIST, [MESSAGE_NAMES.SEARCH_BSKY_USER_ON_BLOCK_PAGE]: ACTION_MODE.BLOCK, + [MESSAGE_NAMES.SEARCH_BSKY_USER_ON_THREADS_PAGE]: ACTION_MODE.FOLLOW, }; const STORAGE_PREFIX = "sky_follower_bridge_storage"; @@ -46,6 +50,7 @@ export const TARGET_URLS_REGEX = { FOLLOW: /https:\/\/(twitter|x)\.com\/[^/]+\/(verified_follow|follow)/, LIST: /^https:\/\/(twitter|x)\.com\/[^/]+\/lists\/[^/]+\/members/, BLOCK: /^https:\/\/(twitter|x)\.com\/settings\/blocked/, + THREADS: /^https:\/\/www\.threads\.net/, } as const; export const MESSAGE_TYPE = { @@ -112,3 +117,8 @@ export const BSKY_PROFILE_LABEL = { } as const; export const DEFAULT_LIST_NAME = "Imported List from X"; + +export const SERVICE_TYPE = { + X: "x", + THREADS: "threads", +} as const; diff --git a/src/lib/hooks/useRetrieveBskyUsers.ts b/src/lib/hooks/useRetrieveBskyUsers.ts index 960bf3c..8e5e09b 100644 --- a/src/lib/hooks/useRetrieveBskyUsers.ts +++ b/src/lib/hooks/useRetrieveBskyUsers.ts @@ -4,25 +4,45 @@ import { useStorage } from "@plasmohq/storage/hook"; import React from "react"; import { P, match } from "ts-pattern"; import { BskyServiceWorkerClient } from "~lib/bskyServiceWorkerClient"; -import { MESSAGE_NAMES, STORAGE_KEYS } from "~lib/constants"; +import { MESSAGE_NAMES, SERVICE_TYPE, STORAGE_KEYS } from "~lib/constants"; import { searchBskyUser } from "~lib/searchBskyUsers"; import type { AbstractService } from "~lib/services/abstractService"; +import { ThreadsService } from "~lib/services/threadsService"; import { XService } from "~lib/services/xService"; -import type { BskyUser, CrawledUserInfo, MessageName } from "~types"; +import type { + BskyUser, + CrawledUserInfo, + MessageName, + ServiceType, +} from "~types"; -const getService = (messageName: MessageName): AbstractService => { +const getServiceType = (messageName: MessageName): ServiceType => { return match(messageName) + .returnType<(typeof SERVICE_TYPE)[keyof typeof SERVICE_TYPE]>() .with( - P.when((name) => - [ - MESSAGE_NAMES.SEARCH_BSKY_USER_ON_FOLLOW_PAGE, - MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE, - MESSAGE_NAMES.SEARCH_BSKY_USER_ON_BLOCK_PAGE, - ].includes(name as MessageName), + P.union( + MESSAGE_NAMES.SEARCH_BSKY_USER_ON_FOLLOW_PAGE, + MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE, + MESSAGE_NAMES.SEARCH_BSKY_USER_ON_BLOCK_PAGE, ), - () => new XService(messageName), + () => SERVICE_TYPE.X, ) - .otherwise(() => new XService(messageName)); + .with( + MESSAGE_NAMES.SEARCH_BSKY_USER_ON_THREADS_PAGE, + () => SERVICE_TYPE.THREADS, + ) + .run(); +}; + +const buildService = ( + serviceType: ServiceType, + messageName: MessageName, +): AbstractService => { + return match(serviceType) + .returnType() + .with(SERVICE_TYPE.X, () => new XService(messageName)) + .with(SERVICE_TYPE.THREADS, () => new ThreadsService(messageName)) + .run(); }; export const useRetrieveBskyUsers = () => { @@ -39,6 +59,9 @@ export const useRetrieveBskyUsers = () => { const [loading, setLoading] = React.useState(true); const [errorMessage, setErrorMessage] = React.useState(""); const [isBottomReached, setIsBottomReached] = React.useState(false); + const [currentService, setCurrentService] = React.useState< + (typeof SERVICE_TYPE)[keyof typeof SERVICE_TYPE] + >(SERVICE_TYPE.X); const [retrievalParams, setRetrievalParams] = React.useState { }>(null); const retrieveBskyUsers = React.useCallback( - async (usersData: CrawledUserInfo[]) => { + async ( + usersData: CrawledUserInfo[], + processExtractedData: (user: CrawledUserInfo) => Promise, + ) => { for (const userData of usersData) { const searchResult = await searchBskyUser({ client: bskyClient.current, userData, }); if (searchResult) { + const processedUser = await processExtractedData(userData); await setUsers((prev) => { if (prev.some((u) => u.did === searchResult.bskyProfile.did)) { return prev; @@ -70,10 +97,10 @@ export const useRetrieveBskyUsers = () => { followingUri: searchResult.bskyProfile.viewer?.following, isBlocking: !!searchResult.bskyProfile.viewer?.blocking, blockingUri: searchResult.bskyProfile.viewer?.blocking, - originalAvatar: userData.originalAvatar, - originalHandle: userData.accountName, - originalDisplayName: userData.displayName, - originalProfileLink: userData.originalProfileLink, + originalAvatar: processedUser.originalAvatar, + originalHandle: processedUser.accountName, + originalDisplayName: processedUser.displayName, + originalProfileLink: processedUser.originalProfileLink, }, ]; }); @@ -85,22 +112,20 @@ export const useRetrieveBskyUsers = () => { const abortControllerRef = React.useRef(null); const startRetrieveLoop = React.useCallback( - async (messageName: MessageName) => { + async (service: AbstractService) => { abortControllerRef.current = new AbortController(); const signal = abortControllerRef.current.signal; let index = 0; - const service = getService(messageName); - // loop until we get to the bottom - while (!isBottomReached) { + while (true) { if (signal.aborted) { break; } const data = service.getCrawledUsers(); - await retrieveBskyUsers(data); + await retrieveBskyUsers(data, service.processExtractedData); const isEnd = await service.performScrollAndCheckEnd(); @@ -117,7 +142,7 @@ export const useRetrieveBskyUsers = () => { } } }, - [retrieveBskyUsers, isBottomReached], + [retrieveBskyUsers], ); const stopRetrieveLoop = React.useCallback(() => { @@ -143,22 +168,34 @@ export const useRetrieveBskyUsers = () => { bskyClient.current = new BskyServiceWorkerClient(session); - startRetrieveLoop(messageName).catch((e) => { + const serviceType = getServiceType(messageName); + setCurrentService(serviceType); + const service = buildService(serviceType, messageName); + + const [isTargetPage, errorMessage] = service.isTargetPage(); + if (!isTargetPage) { + throw new Error(errorMessage); + } + + startRetrieveLoop(service).catch((e) => { console.error(e); setErrorMessage(e.message); setLoading(false); }); + setErrorMessage(""); setLoading(true); await setUsers([]); }, []); const restart = React.useCallback(() => { - startRetrieveLoop(retrievalParams.messageName).catch((e) => { + const service = buildService(currentService, retrievalParams.messageName); + startRetrieveLoop(service).catch((e) => { setErrorMessage(e.message); setLoading(false); }); + setErrorMessage(""); setLoading(true); - }, [retrievalParams, startRetrieveLoop]); + }, [currentService, retrievalParams, startRetrieveLoop]); const isRateLimitError = React.useMemo(() => { // TODO: improve this logic @@ -180,5 +217,6 @@ export const useRetrieveBskyUsers = () => { isSucceeded, isBottomReached, stopRetrieveLoop, + currentService, }; }; diff --git a/src/lib/services/abstractService.ts b/src/lib/services/abstractService.ts index 47d5aa4..4158625 100644 --- a/src/lib/services/abstractService.ts +++ b/src/lib/services/abstractService.ts @@ -1,5 +1,5 @@ import { MESSAGE_NAME_TO_QUERY_PARAM_MAP } from "~lib/constants"; -import type { CrawledUserInfo, MessageName } from "~types"; +import type { BskyUser, CrawledUserInfo, MessageName } from "~types"; export abstract class AbstractService { messageName: MessageName; @@ -10,6 +10,12 @@ export abstract class AbstractService { this.crawledUsers = new Set(); } + abstract isTargetPage(): [boolean, string]; + + abstract processExtractedData( + user: CrawledUserInfo, + ): Promise; + abstract extractUserData(userCell: Element): CrawledUserInfo; getCrawledUsers(): CrawledUserInfo[] { @@ -19,17 +25,18 @@ export abstract class AbstractService { ), ); - const users = Array.from(userCells) - .map((userCell) => this.extractUserData(userCell)) - .filter((user) => { - const isNewUser = !this.crawledUsers.has(user.accountName); - if (isNewUser) { - this.crawledUsers.add(user.accountName); - } - return isNewUser; - }); + const users = Array.from(userCells).map((userCell) => + this.extractUserData(userCell), + ); + const filteredUsers = users.filter((user) => { + const isNewUser = !this.crawledUsers.has(user.accountName); + if (isNewUser) { + this.crawledUsers.add(user.accountName); + } + return isNewUser; + }); - return users; + return filteredUsers; } abstract performScrollAndCheckEnd(): Promise; diff --git a/src/lib/services/threadsService.ts b/src/lib/services/threadsService.ts new file mode 100644 index 0000000..ea54742 --- /dev/null +++ b/src/lib/services/threadsService.ts @@ -0,0 +1,80 @@ +import { findFirstScrollableElements, wait } from "~lib/utils"; +import type { CrawledUserInfo } from "~types"; +import { AbstractService } from "./abstractService"; + +export class ThreadsService extends AbstractService { + async processExtractedData(user: CrawledUserInfo): Promise { + const avatarUrl = user.originalAvatar; + if (avatarUrl) { + try { + const response = await fetch(avatarUrl); + const blob = await response.blob(); + const reader = new FileReader(); + const base64Url = await new Promise((resolve, reject) => { + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = reject; + reader.readAsDataURL(blob); + }); + user.originalAvatar = base64Url; + } catch (error) { + console.error("Failed to convert avatar to base64:", error); + } + } + return user; + } + + isTargetPage(): [boolean, string] { + const isTargetPage = + document.querySelector('[role="dialog"] [aria-label="Followers"]') || + document.querySelector('[role="dialog"] [aria-label="Following"]'); + if (!isTargetPage) { + return [ + false, + "Error: Invalid page. please open the following or followers view.", + ]; + } + return [true, ""]; + } + + extractUserData(userCell: Element): CrawledUserInfo { + const [_accountName, displayName] = + (userCell as HTMLElement).innerText?.split("\n").map((t) => t.trim()) ?? + []; + const accountName = _accountName.replaceAll(".", ""); + const accountNameRemoveUnderscore = accountName.replaceAll("_", ""); // bsky does not allow underscores in handle, so remove them. + const accountNameReplaceUnderscore = accountName.replaceAll("_", "-"); + const avatarElement = userCell.querySelector("img"); + const avatarSrc = avatarElement?.getAttribute("src") ?? ""; + + return { + accountName, + displayName, + accountNameRemoveUnderscore, + accountNameReplaceUnderscore, + bskyHandle: "", + originalAvatar: avatarSrc, + originalProfileLink: `https://www.threads.net/@${_accountName}`, + }; + } + + async performScrollAndCheckEnd(): Promise { + const scrollTarget = findFirstScrollableElements( + document.querySelector('[role="dialog"]') as HTMLElement, + ); + + if (!scrollTarget) { + return true; + } + + const initialScrollHeight = scrollTarget.scrollHeight; + scrollTarget.scrollTop += initialScrollHeight; + + await wait(3000); + + const hasReachedEnd = + scrollTarget.scrollTop + scrollTarget.clientHeight >= + scrollTarget.scrollHeight; + + return hasReachedEnd; + } +} diff --git a/src/lib/services/xService.ts b/src/lib/services/xService.ts index 2567a12..c876496 100644 --- a/src/lib/services/xService.ts +++ b/src/lib/services/xService.ts @@ -19,6 +19,15 @@ export class XService extends AbstractService { super(messageName); } + // X determines the target page based on the URL on the popup side, so it always returns true + isTargetPage(): [boolean, string] { + return [true, ""]; + } + + async processExtractedData(user: CrawledUserInfo): Promise { + return user; + } + extractUserData(userCell: Element): CrawledUserInfo { const anchors = Array.from(userCell.querySelectorAll("a")); const [avatarEl, displayNameEl] = anchors; diff --git a/src/lib/utils.ts b/src/lib/utils.ts index b03b9ba..a04ab3e 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -11,3 +11,28 @@ export const isOneSymbol = (str: string) => { export const wait = (ms: number) => { return new Promise((resolve) => setTimeout(resolve, ms)); }; + +export const findFirstScrollableElements = ( + targetElement: HTMLElement, +): HTMLElement | null => { + const isScrollable = (element: HTMLElement) => { + const style = window.getComputedStyle(element); + const isOverflowScrollable = + style.overflowY === "auto" || style.overflowY === "scroll"; + const canScrollVertically = element.scrollHeight > element.clientHeight; + + return isOverflowScrollable && canScrollVertically; + }; + + const allElements = targetElement.querySelectorAll("*"); + const scrollableElements: HTMLElement[] = []; + + for (const element of allElements) { + const htmlElement = element as HTMLElement; + if (isScrollable(htmlElement)) { + scrollableElements.push(htmlElement); + } + } + + return scrollableElements[0] ?? null; +}; diff --git a/src/popup.tsx b/src/popup.tsx index 73ef140..2a6b84c 100644 --- a/src/popup.tsx +++ b/src/popup.tsx @@ -116,7 +116,7 @@ function IndexPopup() { if (!Object.values(TARGET_URLS_REGEX).some((r) => r.test(currentUrl))) { setErrorMessage( - "Error: Invalid page. please open the 𝕏 following or blocking or list page.", + "Error: Invalid page. please open the target page.", DOCUMENT_LINK.PAGE_ERROR, ); return; @@ -135,6 +135,10 @@ function IndexPopup() { P.when((url) => TARGET_URLS_REGEX.LIST.test(url)), () => MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE, ) + .with( + P.when((url) => TARGET_URLS_REGEX.THREADS.test(url)), + () => MESSAGE_NAMES.SEARCH_BSKY_USER_ON_THREADS_PAGE, + ) .run(); await chrome.storage.local.set({ @@ -168,13 +172,20 @@ function IndexPopup() { } return; } + await chrome.storage.local.set({ [STORAGE_KEYS.BSKY_CLIENT_SESSION]: session, }); - await clearPasswordFromStorage(); - await sendToContentScript({ + + const { hasError, message: errorMessage } = await sendToContentScript({ name: messageName, }); + if (hasError) { + setErrorMessage(errorMessage, DOCUMENT_LINK.OTHER_ERROR); + return; + } + + await clearPasswordFromStorage(); await saveShowAuthFactorTokenInputToStorage(false); window.close(); } catch (e) { diff --git a/src/types.ts b/src/types.ts index 714ed1c..07a9856 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,8 @@ -import type { BSKY_USER_MATCH_TYPE, MESSAGE_NAMES } from "~lib/constants"; +import type { + BSKY_USER_MATCH_TYPE, + MESSAGE_NAMES, + SERVICE_TYPE, +} from "~lib/constants"; export type MatchType = (typeof BSKY_USER_MATCH_TYPE)[keyof typeof BSKY_USER_MATCH_TYPE]; @@ -38,3 +42,5 @@ export type CrawledUserInfo = { originalAvatar: string; originalProfileLink: string; }; + +export type ServiceType = (typeof SERVICE_TYPE)[keyof typeof SERVICE_TYPE];