Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | import { findCAS } from '@/helpers/cas';
import { HttpError } from '@/helpers/exceptions';
import { sleep } from '@/helpers/utils';
import { ProductBuilder } from '@/utils/ProductBuilder';
import { isHtmlResponse } from '@/utils/typeGuards/common';
import { SupplierBaseMagento2 } from './SupplierBaseMagento2';
import { HttpStatus } from '@/constants/httpStatus';
/** Outcome of a single product-page fetch attempt. */
export type PageFetchResult =
| { status: 'ok'; html?: string }
// `httpStatus` carries the HTTP status when the failure was an HttpError (e.g. 429, 404);
// undefined for non-HTTP failures (network error, non-HTML response).
| { status: 'error'; httpStatus?: number };
/**
* SupplierAladdinSci class that extends SupplierBaseMagento2 and implements AsyncIterable<Product>.
*
* @remarks
* AladdinSci is a Magento 2 storefront that exposes the public unauthenticated
* GraphQL endpoint at `/graphql`. The `Store: us_en` header (default) selects
* the US storefront. The GraphQL search response only carries the product
* picture for the new chemical-identifier fields, so {@link getProductData}
* fetches each product's HTML page and scrapes the remaining data (SMILES,
* IUPAC name, InChIKey, INCHI, PubChem CID, molecular weight, SDS, spec sheet).
*
* @category Suppliers
* @source
*/
export class SupplierAladdinSci extends SupplierBaseMagento2 implements ISupplier {
// Name of supplier (for display purposes)
public static readonly supplierName: string = 'AladdinSci';
// Base URL for HTTP(s) requests
public static readonly baseURL: string = 'https://www.aladdinsci.com';
// Shipping scope for AladdinSci
public static readonly shipping: ShippingRange = 'worldwide';
// The country code of the supplier.
public static readonly country: CountryCode = 'US';
// The payment methods accepted by the supplier.
public static readonly paymentMethods: PaymentMethod[] = ['mastercard', 'visa'];
/** Base unit (n) for the escalating rate-limit backoff: pause = n, then 2n, 3n, … */
private readonly backoffBaseDelayMs: number = 1000;
/** Max attempts for a single product page before giving up (the product is still listed). */
private readonly maxPageFetchAttempts: number = 5;
/** True while one task is probing the server after a 429; all other page fetches wait. */
private backoffActive: boolean = false;
/**
* Number of 429 backoff pauses taken so far this search; multiplies {@link backoffBaseDelayMs}
* so consecutive 429s wait progressively longer (n, 2n, 3n, …). Resets per search since a fresh
* supplier instance is created for each one.
*/
private backoffLevel: number = 0;
/** Gate awaited before each page fetch; a pending promise while a backoff probe runs. */
private backoffGate: Promise<void> = Promise.resolve();
/** Resolver that lifts {@link backoffGate}; defined only while the gate is closed. */
private liftBackoffGate?: () => void;
/**
* Enriches a product by fetching its HTML product page (the permalink) and scraping the
* fields that aren't available from the GraphQL search response. Wrapped in the shared
* caching layer so each product page is fetched at most once per cache TTL.
*
* @param product - The product builder to enrich
* @returns Promise resolving to the enriched product builder or void
* @example
* ```typescript
* const enriched = await this.getProductData(builder);
* if (enriched) {
* console.log((await enriched.build())?.smiles);
* }
* ```
* @source
*/
protected async getProductData(
product: ProductBuilder<Product>,
): Promise<ProductBuilder<Product> | void> {
return this.getProductDataWithCache(product, async (builder) => {
const permalink = builder.get('permalink') ?? builder.get('url');
if (typeof permalink !== 'string') {
return builder;
}
// Fetch the product page with shared 429 backoff. Any failure (rate-limit retries
// exhausted, network error, etc.) yields undefined, so the product is still listed —
// just without the scraped data. The builder validates sufficiency at build time.
const html = await this.fetchProductPage(permalink);
if (!html) {
return builder;
}
const doc = new DOMParser().parseFromString(html, 'text/html');
// If there is no formula, then this grabs random strings that are not a formula.
// const synonyms = this.getSynonyms(doc);
// if (synonyms) {
// this.logger.debug(`Found Synonyms: ${synonyms}`, { synonyms });
// const formula = findFormulaInText(synonyms);
// this.logger.debug(`Found Formula: ${formula}`, { formula });
// builder.setFormula(formula ?? undefined);
// }
builder
.setCAS(this.casNumber(doc))
.setSmiles(this.cellText(doc, 'Isomeric SMILES'))
.setIupacName(this.cellText(doc, 'IUPAC Name'))
.setInChIKey(this.cellText(doc, 'InChIKey'))
.setInChI(this.cellText(doc, 'INCHI'))
.setPubchemId(this.cellText(doc, 'PubChem CID'))
.setMoleweight(this.molecularWeight(doc))
.setPurity(this.getPurity(doc))
.setSDSUrl(this.safetyGridLink(doc, 'SDS'))
.setSpecSheetUrl(this.safetyGridLink(doc, 'Specification Sheet'));
return builder;
});
}
private getSynonyms(doc: Document): string | undefined {
return (
Array.from(doc.querySelectorAll('#specifications .panel-body > .spec-table > .spec-row'))
?.find((r) => r.querySelector('.k')?.textContent?.toLowerCase().trim() == 'synonyms')
?.querySelector('.v')
?.textContent?.trim() ?? undefined
);
}
/**
* Fetches a product page's HTML, applying shared rate-limit (429) backoff across all
* concurrent product-page requests for this supplier. On a 429 the first task to see it
* becomes the "prober" (see {@link probeAfterRateLimit}): it pauses every other page fetch,
* waits, then retries a single request — resuming the rest only once a request succeeds.
* Other tasks that hit a 429 meanwhile simply wait for the probe to clear. Any failure
* (retries exhausted or a non-429 error) resolves to undefined so the product is still listed;
* the failing status is recorded for {@link shouldCacheProductData}.
*
* @param url - The absolute product page URL to fetch
* @returns The page HTML, or undefined when the page could not be retrieved
* @example
* ```typescript
* const html = await this.fetchProductPage("https://www.aladdinsci.com/us_en/x.html");
* ```
* @source
*/
private async fetchProductPage(url: string): Promise<string | undefined> {
for (let attempt = 0; attempt < this.maxPageFetchAttempts; attempt++) {
// Bail immediately if the search was aborted (e.g. supplierSearchTimeBudgetSec elapsed).
if (this.controller.signal.aborted) {
return undefined;
}
// Wait while another task is probing the server after a 429.
await this.backoffGate;
const result = await this.attemptPageFetch(url);
if (result.status === 'ok') {
this.failedFetchStatuses.delete(url);
return result.html;
}
this.recordFetchFailure(url, result.httpStatus);
// Only a 429 triggers the shared backoff; any other failure ends this product's fetch.
if (result.httpStatus !== HttpStatus.TOO_MANY_REQUESTS) {
return undefined;
}
// Rate limited. If a probe is already running, loop and wait on the gate; otherwise
// become the prober (which pauses everyone else, waits, and retries one at a time).
if (this.backoffActive) {
continue;
}
return await this.probeAfterRateLimit(url);
}
return undefined;
}
/**
* Single-flight backoff probe. Pauses all other product-page fetches, then repeatedly waits an
* escalating interval ({@link backoffBaseDelayMs} × the running {@link backoffLevel}, i.e.
* n, 2n, 3n, …) and retries one request until it succeeds, hits a non-429 error, or exhausts
* {@link maxPageFetchAttempts}. The gate is always lifted on exit so paused tasks resume.
*
* @param url - The absolute product page URL to retry
* @returns The page HTML on success, or undefined when the probe gives up
* @example
* ```typescript
* const html = await this.probeAfterRateLimit("https://www.aladdinsci.com/us_en/x.html");
* ```
* @source
*/
private async probeAfterRateLimit(url: string): Promise<string | undefined> {
this.backoffActive = true;
this.closeBackoffGate();
try {
for (let attempt = 0; attempt < this.maxPageFetchAttempts; attempt++) {
if (this.controller.signal.aborted) {
return undefined;
}
// Escalate the pause on each consecutive 429: n, then 2n, then 3n, …
this.backoffLevel++;
await sleep(this.backoffBaseDelayMs * this.backoffLevel);
const result = await this.attemptPageFetch(url);
if (result.status === 'ok') {
this.failedFetchStatuses.delete(url);
return result.html;
}
this.recordFetchFailure(url, result.httpStatus);
if (result.httpStatus !== HttpStatus.TOO_MANY_REQUESTS) {
return undefined;
}
// Still rate limited: wait another (longer) interval and probe again.
}
return undefined;
} finally {
this.backoffActive = false;
this.openBackoffGate();
}
}
/**
* Performs a single product-page GET, classifying the outcome so the backoff loop can react.
* On failure, the HTTP status (when the error was an HttpError) is included so the caller can
* branch on 429 and record it for cache decisions.
*
* @param url - The absolute product page URL to fetch
* @returns The classified fetch result
* @example
* ```typescript
* const result = await this.attemptPageFetch(url); // { status: "ok", html: "<!doctype..." }
* ```
* @source
*/
private async attemptPageFetch(url: string): Promise<PageFetchResult> {
try {
// rethrowErrors surfaces the HttpError (e.g. 429) instead of swallowing it.
const response = await this.httpGet({ path: url, rethrowErrors: true });
if (!response || !isHtmlResponse(response)) {
return { status: 'error' };
}
return { status: 'ok', html: await response.text() };
} catch (error: unknown) {
if (error instanceof HttpError) {
return { status: 'error', httpStatus: error.status };
}
return { status: 'error' };
}
}
/**
* Closes the backoff gate so subsequent page fetches wait until it is lifted. Idempotent.
* @returns void
* @source
*/
private closeBackoffGate(): void {
if (this.liftBackoffGate) {
return;
}
this.backoffGate = new Promise<void>((resolve) => {
this.liftBackoffGate = resolve;
});
}
/**
* Lifts the backoff gate, resuming any paused page fetches. Idempotent.
* @returns void
* @source
*/
private openBackoffGate(): void {
if (!this.liftBackoffGate) {
return;
}
this.liftBackoffGate();
this.liftBackoffGate = undefined;
this.backoffGate = Promise.resolve();
}
/**
* Extracts the CAS number from the product page. Prefers the `.title-meta` "CAS:" span, falling
* back to the page title (`.title-row h1`), which also carries the CAS (e.g. "… CAS No.10017-56-8").
* The value is run through {@link findCAS}, so only a checksum-valid CAS is returned.
*
* @param doc - The parsed product-page document
* @returns The CAS number, or undefined when none is found
* @example
* ```typescript
* this.casNumber(doc); // "10017-56-8"
* ```
* @source
*/
private casNumber(doc: Document): string | undefined {
const casSpan = Array.from(doc.querySelectorAll('.title-meta > span')).find((span) =>
(span.querySelector('strong')?.textContent ?? '').trim().toLowerCase().startsWith('cas'),
);
const title = doc.querySelector('.title-row h1')?.textContent;
return findCAS(casSpan?.textContent ?? '') || findCAS(title ?? '') || undefined;
}
/**
* Reads the trimmed text of a product-detail table cell, keyed by its `data-th` attribute.
*
* @param doc - The parsed product-page document
* @param label - The `data-th` value identifying the cell (e.g. "IUPAC Name")
* @returns The trimmed cell text, or undefined when the cell is missing or empty
* @example
* ```typescript
* this.cellText(doc, "IUPAC Name"); // "dipotassium;oxalate"
* ```
* @source
*/
private cellText(doc: Document, label: string): string | undefined {
const text = doc.querySelector(`td[data-th="${label}"]`)?.textContent?.trim();
return text && text.length > 0 ? text : undefined;
}
/**
* Extracts the molecular weight from the product-detail table. The cell may include a unit
* (e.g. "254.32 g/mol"), so only the leading numeric portion is returned for `setMoleweight`.
*
* @param doc - The parsed product-page document
* @returns The numeric molecular weight as a string, or undefined when unavailable
* @example
* ```typescript
* this.molecularWeight(doc); // "254.32"
* ```
* @source
*/
private molecularWeight(doc: Document): string | undefined {
const raw = this.cellText(doc, 'Molecular Weight');
return raw?.match(/\d+(?:\.\d+)?/)?.[0];
}
/**
* Extracts the purity from the `#specifications` table. Prefers the explicit "Purity" row; if
* absent, falls back to the combined "Specifications & Purity" row but only when its value is a
* percentage. Returns the raw value with any qualifier intact (e.g. "≥99%") — `setPurity`
* validates and normalizes it.
*
* @param doc - The parsed product-page document
* @returns The purity value as written (e.g. "≥99%"), or undefined when unavailable
* @example
* ```typescript
* this.getPurity(doc); // "≥99%"
* ```
* @source
*/
private getPurity(doc: Document): string | undefined {
const valueFor = (label: string): string | undefined => {
for (const row of doc.querySelectorAll('#specifications .spec-row')) {
if (row.querySelector('.k')?.textContent?.trim().toLowerCase() === label.toLowerCase()) {
return row.querySelector('.v')?.textContent?.trim();
}
}
return undefined;
};
// Prefer the dedicated "Purity" row; otherwise use "Specifications & Purity" only if it's a %.
const raw = valueFor('Purity');
if (raw !== undefined) {
return raw;
}
const fallback = valueFor('Specifications & Purity');
return fallback?.includes('%') ? fallback : undefined;
}
/**
* Finds the download link inside the `.safety-grid` whose item heading contains the given text.
* Placeholder anchors (e.g. `href="#"`) are ignored so only real document URLs are returned.
*
* @param doc - The parsed product-page document
* @param heading - Text to match within a safety item's `<h4>` (e.g. "SDS", "Specification Sheet")
* @returns The matching document URL, or undefined when no real link is found
* @example
* ```typescript
* this.safetyGridLink(doc, "SDS"); // "https://.../P693472-SCI_....pdf"
* ```
* @source
*/
private safetyGridLink(doc: Document, heading: string): string | undefined {
const grid = doc.querySelector('.safety-grid');
if (!grid) {
return undefined;
}
for (const item of grid.querySelectorAll('.safety-item')) {
if (item.querySelector('h4')?.textContent?.includes(heading)) {
const href = item.querySelector('a')?.getAttribute('href')?.trim();
return href && !href.startsWith('#') ? href : undefined;
}
}
return undefined;
}
}
|