feat: support threds

This commit is contained in:
kawamataryo 2024-11-22 21:33:21 +09:00
parent 0d88286ffd
commit 6d6a2d2a11
10 changed files with 244 additions and 44 deletions

View File

@ -67,7 +67,8 @@
"host_permissions": [
"https://bsky.social/*",
"https://twitter.com/*",
"https://x.com/*"
"https://x.com/*",
"https://www.threads.net/*"
],
"browser_specific_settings": {
"gecko": {

View File

@ -4,14 +4,19 @@ import { Storage } from "@plasmohq/storage";
import { useStorage } from "@plasmohq/storage/hook";
import type { PlasmoCSConfig } from "plasmo";
import React from "react";
import { match } from "ts-pattern";
import AlertError from "~lib/components/AlertError";
import LoadingCards from "~lib/components/LoadingCards";
import Modal from "~lib/components/Modal";
import { MESSAGE_NAMES, STORAGE_KEYS } from "~lib/constants";
import { MESSAGE_NAMES, SERVICE_TYPE, STORAGE_KEYS } from "~lib/constants";
import { useRetrieveBskyUsers } from "~lib/hooks/useRetrieveBskyUsers";
export const config: PlasmoCSConfig = {
matches: ["https://twitter.com/*", "https://x.com/*"],
matches: [
"https://twitter.com/*",
"https://x.com/*",
"https://www.threads.net/*",
],
all_frames: true,
};
@ -31,6 +36,7 @@ const App = () => {
restart,
isBottomReached,
errorMessage,
currentService,
} = useRetrieveBskyUsers();
const [isModalOpen, setIsModalOpen] = React.useState(false);
@ -89,13 +95,20 @@ const App = () => {
stopRetrieveLoop();
};
const serviceName = React.useMemo(() => {
return match(currentService)
.with(SERVICE_TYPE.X, () => "X")
.with(SERVICE_TYPE.THREADS, () => "Threads")
.exhaustive();
}, [currentService]);
return (
<>
<Modal open={isModalOpen} onClose={closeModal}>
<div className="flex flex-col gap-2 items-center">
{loading && (
<p className="text-lg font-bold">
Scanning 𝕏 users to find bsky users...
Scanning {serviceName} users to find bsky users...
</p>
)}
<p className="text-2xl font-bold">

View File

@ -3,18 +3,21 @@ export const MESSAGE_NAMES = {
SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE:
"search_bsky_user_on_list_members_page",
SEARCH_BSKY_USER_ON_BLOCK_PAGE: "search_bsky_user_on_block_page",
SEARCH_BSKY_USER_ON_THREADS_PAGE: "search_bsky_user_on_threads_page",
} as const;
export const QUERY_PARAMS = {
FOLLOW: '[data-testid="primaryColumn"] [data-testid="UserCell"]',
BLOCK: '[data-testid="UserCell"]',
LIST: '[data-testid="cellInnerDiv"] [data-testid="UserCell"]',
THREADS: '[data-pressable-container="true"]',
} as const;
export const MESSAGE_NAME_TO_QUERY_PARAM_MAP = {
[MESSAGE_NAMES.SEARCH_BSKY_USER_ON_FOLLOW_PAGE]: QUERY_PARAMS.FOLLOW,
[MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE]: QUERY_PARAMS.LIST,
[MESSAGE_NAMES.SEARCH_BSKY_USER_ON_BLOCK_PAGE]: QUERY_PARAMS.BLOCK,
[MESSAGE_NAMES.SEARCH_BSKY_USER_ON_THREADS_PAGE]: QUERY_PARAMS.THREADS,
};
export const ACTION_MODE = {
@ -28,6 +31,7 @@ export const MESSAGE_NAME_TO_ACTION_MODE_MAP = {
[MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE]:
ACTION_MODE.IMPORT_LIST,
[MESSAGE_NAMES.SEARCH_BSKY_USER_ON_BLOCK_PAGE]: ACTION_MODE.BLOCK,
[MESSAGE_NAMES.SEARCH_BSKY_USER_ON_THREADS_PAGE]: ACTION_MODE.FOLLOW,
};
const STORAGE_PREFIX = "sky_follower_bridge_storage";
@ -46,6 +50,7 @@ export const TARGET_URLS_REGEX = {
FOLLOW: /https:\/\/(twitter|x)\.com\/[^/]+\/(verified_follow|follow)/,
LIST: /^https:\/\/(twitter|x)\.com\/[^/]+\/lists\/[^/]+\/members/,
BLOCK: /^https:\/\/(twitter|x)\.com\/settings\/blocked/,
THREADS: /^https:\/\/www\.threads\.net/,
} as const;
export const MESSAGE_TYPE = {
@ -112,3 +117,8 @@ export const BSKY_PROFILE_LABEL = {
} as const;
export const DEFAULT_LIST_NAME = "Imported List from X";
export const SERVICE_TYPE = {
X: "x",
THREADS: "threads",
} as const;

View File

@ -4,25 +4,45 @@ import { useStorage } from "@plasmohq/storage/hook";
import React from "react";
import { P, match } from "ts-pattern";
import { BskyServiceWorkerClient } from "~lib/bskyServiceWorkerClient";
import { MESSAGE_NAMES, STORAGE_KEYS } from "~lib/constants";
import { MESSAGE_NAMES, SERVICE_TYPE, STORAGE_KEYS } from "~lib/constants";
import { searchBskyUser } from "~lib/searchBskyUsers";
import type { AbstractService } from "~lib/services/abstractService";
import { ThreadsService } from "~lib/services/threadsService";
import { XService } from "~lib/services/xService";
import type { BskyUser, CrawledUserInfo, MessageName } from "~types";
import type {
BskyUser,
CrawledUserInfo,
MessageName,
ServiceType,
} from "~types";
const getService = (messageName: MessageName): AbstractService => {
const getServiceType = (messageName: MessageName): ServiceType => {
return match(messageName)
.returnType<(typeof SERVICE_TYPE)[keyof typeof SERVICE_TYPE]>()
.with(
P.when((name) =>
[
MESSAGE_NAMES.SEARCH_BSKY_USER_ON_FOLLOW_PAGE,
MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE,
MESSAGE_NAMES.SEARCH_BSKY_USER_ON_BLOCK_PAGE,
].includes(name as MessageName),
P.union(
MESSAGE_NAMES.SEARCH_BSKY_USER_ON_FOLLOW_PAGE,
MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE,
MESSAGE_NAMES.SEARCH_BSKY_USER_ON_BLOCK_PAGE,
),
() => new XService(messageName),
() => SERVICE_TYPE.X,
)
.otherwise(() => new XService(messageName));
.with(
MESSAGE_NAMES.SEARCH_BSKY_USER_ON_THREADS_PAGE,
() => SERVICE_TYPE.THREADS,
)
.run();
};
const buildService = (
serviceType: ServiceType,
messageName: MessageName,
): AbstractService => {
return match(serviceType)
.returnType<AbstractService>()
.with(SERVICE_TYPE.X, () => new XService(messageName))
.with(SERVICE_TYPE.THREADS, () => new ThreadsService(messageName))
.run();
};
export const useRetrieveBskyUsers = () => {
@ -39,6 +59,9 @@ export const useRetrieveBskyUsers = () => {
const [loading, setLoading] = React.useState(true);
const [errorMessage, setErrorMessage] = React.useState("");
const [isBottomReached, setIsBottomReached] = React.useState(false);
const [currentService, setCurrentService] = React.useState<
(typeof SERVICE_TYPE)[keyof typeof SERVICE_TYPE]
>(SERVICE_TYPE.X);
const [retrievalParams, setRetrievalParams] = React.useState<null | {
session: AtpSessionData;
@ -46,13 +69,17 @@ export const useRetrieveBskyUsers = () => {
}>(null);
const retrieveBskyUsers = React.useCallback(
async (usersData: CrawledUserInfo[]) => {
async (
usersData: CrawledUserInfo[],
processExtractedData: (user: CrawledUserInfo) => Promise<CrawledUserInfo>,
) => {
for (const userData of usersData) {
const searchResult = await searchBskyUser({
client: bskyClient.current,
userData,
});
if (searchResult) {
const processedUser = await processExtractedData(userData);
await setUsers((prev) => {
if (prev.some((u) => u.did === searchResult.bskyProfile.did)) {
return prev;
@ -70,10 +97,10 @@ export const useRetrieveBskyUsers = () => {
followingUri: searchResult.bskyProfile.viewer?.following,
isBlocking: !!searchResult.bskyProfile.viewer?.blocking,
blockingUri: searchResult.bskyProfile.viewer?.blocking,
originalAvatar: userData.originalAvatar,
originalHandle: userData.accountName,
originalDisplayName: userData.displayName,
originalProfileLink: userData.originalProfileLink,
originalAvatar: processedUser.originalAvatar,
originalHandle: processedUser.accountName,
originalDisplayName: processedUser.displayName,
originalProfileLink: processedUser.originalProfileLink,
},
];
});
@ -85,22 +112,20 @@ export const useRetrieveBskyUsers = () => {
const abortControllerRef = React.useRef<AbortController | null>(null);
const startRetrieveLoop = React.useCallback(
async (messageName: MessageName) => {
async (service: AbstractService) => {
abortControllerRef.current = new AbortController();
const signal = abortControllerRef.current.signal;
let index = 0;
const service = getService(messageName);
// loop until we get to the bottom
while (!isBottomReached) {
while (true) {
if (signal.aborted) {
break;
}
const data = service.getCrawledUsers();
await retrieveBskyUsers(data);
await retrieveBskyUsers(data, service.processExtractedData);
const isEnd = await service.performScrollAndCheckEnd();
@ -117,7 +142,7 @@ export const useRetrieveBskyUsers = () => {
}
}
},
[retrieveBskyUsers, isBottomReached],
[retrieveBskyUsers],
);
const stopRetrieveLoop = React.useCallback(() => {
@ -143,22 +168,34 @@ export const useRetrieveBskyUsers = () => {
bskyClient.current = new BskyServiceWorkerClient(session);
startRetrieveLoop(messageName).catch((e) => {
const serviceType = getServiceType(messageName);
setCurrentService(serviceType);
const service = buildService(serviceType, messageName);
const [isTargetPage, errorMessage] = service.isTargetPage();
if (!isTargetPage) {
throw new Error(errorMessage);
}
startRetrieveLoop(service).catch((e) => {
console.error(e);
setErrorMessage(e.message);
setLoading(false);
});
setErrorMessage("");
setLoading(true);
await setUsers([]);
}, []);
const restart = React.useCallback(() => {
startRetrieveLoop(retrievalParams.messageName).catch((e) => {
const service = buildService(currentService, retrievalParams.messageName);
startRetrieveLoop(service).catch((e) => {
setErrorMessage(e.message);
setLoading(false);
});
setErrorMessage("");
setLoading(true);
}, [retrievalParams, startRetrieveLoop]);
}, [currentService, retrievalParams, startRetrieveLoop]);
const isRateLimitError = React.useMemo(() => {
// TODO: improve this logic
@ -180,5 +217,6 @@ export const useRetrieveBskyUsers = () => {
isSucceeded,
isBottomReached,
stopRetrieveLoop,
currentService,
};
};

View File

@ -1,5 +1,5 @@
import { MESSAGE_NAME_TO_QUERY_PARAM_MAP } from "~lib/constants";
import type { CrawledUserInfo, MessageName } from "~types";
import type { BskyUser, CrawledUserInfo, MessageName } from "~types";
export abstract class AbstractService {
messageName: MessageName;
@ -10,6 +10,12 @@ export abstract class AbstractService {
this.crawledUsers = new Set();
}
abstract isTargetPage(): [boolean, string];
abstract processExtractedData(
user: CrawledUserInfo,
): Promise<CrawledUserInfo>;
abstract extractUserData(userCell: Element): CrawledUserInfo;
getCrawledUsers(): CrawledUserInfo[] {
@ -19,17 +25,18 @@ export abstract class AbstractService {
),
);
const users = Array.from(userCells)
.map((userCell) => this.extractUserData(userCell))
.filter((user) => {
const isNewUser = !this.crawledUsers.has(user.accountName);
if (isNewUser) {
this.crawledUsers.add(user.accountName);
}
return isNewUser;
});
const users = Array.from(userCells).map((userCell) =>
this.extractUserData(userCell),
);
const filteredUsers = users.filter((user) => {
const isNewUser = !this.crawledUsers.has(user.accountName);
if (isNewUser) {
this.crawledUsers.add(user.accountName);
}
return isNewUser;
});
return users;
return filteredUsers;
}
abstract performScrollAndCheckEnd(): Promise<boolean>;

View File

@ -0,0 +1,80 @@
import { findFirstScrollableElements, wait } from "~lib/utils";
import type { CrawledUserInfo } from "~types";
import { AbstractService } from "./abstractService";
export class ThreadsService extends AbstractService {
async processExtractedData(user: CrawledUserInfo): Promise<CrawledUserInfo> {
const avatarUrl = user.originalAvatar;
if (avatarUrl) {
try {
const response = await fetch(avatarUrl);
const blob = await response.blob();
const reader = new FileReader();
const base64Url = await new Promise<string>((resolve, reject) => {
reader.onloadend = () => resolve(reader.result as string);
reader.onerror = reject;
reader.readAsDataURL(blob);
});
user.originalAvatar = base64Url;
} catch (error) {
console.error("Failed to convert avatar to base64:", error);
}
}
return user;
}
isTargetPage(): [boolean, string] {
const isTargetPage =
document.querySelector('[role="dialog"] [aria-label="Followers"]') ||
document.querySelector('[role="dialog"] [aria-label="Following"]');
if (!isTargetPage) {
return [
false,
"Error: Invalid page. please open the following or followers view.",
];
}
return [true, ""];
}
extractUserData(userCell: Element): CrawledUserInfo {
const [_accountName, displayName] =
(userCell as HTMLElement).innerText?.split("\n").map((t) => t.trim()) ??
[];
const accountName = _accountName.replaceAll(".", "");
const accountNameRemoveUnderscore = accountName.replaceAll("_", ""); // bsky does not allow underscores in handle, so remove them.
const accountNameReplaceUnderscore = accountName.replaceAll("_", "-");
const avatarElement = userCell.querySelector("img");
const avatarSrc = avatarElement?.getAttribute("src") ?? "";
return {
accountName,
displayName,
accountNameRemoveUnderscore,
accountNameReplaceUnderscore,
bskyHandle: "",
originalAvatar: avatarSrc,
originalProfileLink: `https://www.threads.net/@${_accountName}`,
};
}
async performScrollAndCheckEnd(): Promise<boolean> {
const scrollTarget = findFirstScrollableElements(
document.querySelector('[role="dialog"]') as HTMLElement,
);
if (!scrollTarget) {
return true;
}
const initialScrollHeight = scrollTarget.scrollHeight;
scrollTarget.scrollTop += initialScrollHeight;
await wait(3000);
const hasReachedEnd =
scrollTarget.scrollTop + scrollTarget.clientHeight >=
scrollTarget.scrollHeight;
return hasReachedEnd;
}
}

View File

@ -19,6 +19,15 @@ export class XService extends AbstractService {
super(messageName);
}
// X determines the target page based on the URL on the popup side, so it always returns true
isTargetPage(): [boolean, string] {
return [true, ""];
}
async processExtractedData(user: CrawledUserInfo): Promise<CrawledUserInfo> {
return user;
}
extractUserData(userCell: Element): CrawledUserInfo {
const anchors = Array.from(userCell.querySelectorAll("a"));
const [avatarEl, displayNameEl] = anchors;

View File

@ -11,3 +11,28 @@ export const isOneSymbol = (str: string) => {
export const wait = (ms: number) => {
return new Promise((resolve) => setTimeout(resolve, ms));
};
export const findFirstScrollableElements = (
targetElement: HTMLElement,
): HTMLElement | null => {
const isScrollable = (element: HTMLElement) => {
const style = window.getComputedStyle(element);
const isOverflowScrollable =
style.overflowY === "auto" || style.overflowY === "scroll";
const canScrollVertically = element.scrollHeight > element.clientHeight;
return isOverflowScrollable && canScrollVertically;
};
const allElements = targetElement.querySelectorAll("*");
const scrollableElements: HTMLElement[] = [];
for (const element of allElements) {
const htmlElement = element as HTMLElement;
if (isScrollable(htmlElement)) {
scrollableElements.push(htmlElement);
}
}
return scrollableElements[0] ?? null;
};

View File

@ -116,7 +116,7 @@ function IndexPopup() {
if (!Object.values(TARGET_URLS_REGEX).some((r) => r.test(currentUrl))) {
setErrorMessage(
"Error: Invalid page. please open the 𝕏 following or blocking or list page.",
"Error: Invalid page. please open the target page.",
DOCUMENT_LINK.PAGE_ERROR,
);
return;
@ -135,6 +135,10 @@ function IndexPopup() {
P.when((url) => TARGET_URLS_REGEX.LIST.test(url)),
() => MESSAGE_NAMES.SEARCH_BSKY_USER_ON_LIST_MEMBERS_PAGE,
)
.with(
P.when((url) => TARGET_URLS_REGEX.THREADS.test(url)),
() => MESSAGE_NAMES.SEARCH_BSKY_USER_ON_THREADS_PAGE,
)
.run();
await chrome.storage.local.set({
@ -168,13 +172,20 @@ function IndexPopup() {
}
return;
}
await chrome.storage.local.set({
[STORAGE_KEYS.BSKY_CLIENT_SESSION]: session,
});
await clearPasswordFromStorage();
await sendToContentScript({
const { hasError, message: errorMessage } = await sendToContentScript({
name: messageName,
});
if (hasError) {
setErrorMessage(errorMessage, DOCUMENT_LINK.OTHER_ERROR);
return;
}
await clearPasswordFromStorage();
await saveShowAuthFactorTokenInputToStorage(false);
window.close();
} catch (e) {

View File

@ -1,4 +1,8 @@
import type { BSKY_USER_MATCH_TYPE, MESSAGE_NAMES } from "~lib/constants";
import type {
BSKY_USER_MATCH_TYPE,
MESSAGE_NAMES,
SERVICE_TYPE,
} from "~lib/constants";
export type MatchType =
(typeof BSKY_USER_MATCH_TYPE)[keyof typeof BSKY_USER_MATCH_TYPE];
@ -38,3 +42,5 @@ export type CrawledUserInfo = {
originalAvatar: string;
originalProfileLink: string;
};
export type ServiceType = (typeof SERVICE_TYPE)[keyof typeof SERVICE_TYPE];