Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
232 changes: 216 additions & 16 deletions plugins/english/lncrawler.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { fetchApi } from '@libs/fetch';
import { Plugin } from '@/types/plugin';
import { NovelStatus } from '@libs/novelStatus';
import { Filters, FilterTypes } from '@libs/filterInputs';

// LNCrawler — https://lncrawler.monster
// The web frontend is a React SPA; all data comes from its public JSON
Expand Down Expand Up @@ -206,9 +207,36 @@ function parseNovelSources(data: unknown): NovelSources {
* is English). Falls back to the largest source overall when the novel
* has no English source, so the novel still opens.
*/
function chooseSource(detail: NovelSources): SourceOption {
/**
* Pick the source the plugin reads through. When the catalog was
* filtered to languages, those are carried in the novel path
* (`slug?lang=a,b`) and a source in one of them wins — first listed
* language first. Without them this is: the English source with the
* most available chapters (the site's preferred source wins ties when
* it is English), falling back to the largest source overall when the
* novel has no English source, so the novel still opens.
*/
function chooseSource(
detail: NovelSources,
preferLangs: string[] = [],
): SourceOption {
let bestEn: SourceOption | undefined;
let bestAny: SourceOption | undefined;
for (const lang of preferLangs) {
let bestInLang: SourceOption | undefined;
for (const s of detail.sources) {
if (s.available <= 0 || s.language !== lang) continue;
if (
!bestInLang ||
s.available > bestInLang.available ||
(s.available === bestInLang.available &&
s.slug === detail.preferredSlug)
) {
bestInLang = s;
}
}
if (bestInLang) return bestInLang;
}
for (const s of detail.sources) {
if (s.available <= 0) continue;
if (!bestAny || s.available > bestAny.available) bestAny = s;
Expand Down Expand Up @@ -304,40 +332,192 @@ function parseChaptersPage(data: unknown): ChaptersPage {

/**
* Chapter body HTML. The API prefixes the body with an <h1> of the
* chapter title — the app already shows the title, so it is dropped.
* chapter title — which includes the chapter number on LNCrawler.
* Keeping it gives in-page headings showing title and number, so it
* is left in place (only executable content is stripped below).
*/
function parseChapterBody(data: unknown): string {
const res = data as { body?: string };
if (typeof res.body !== 'string') {
throw new Error('Chapter response had an unexpected shape');
}
const withoutTitle = res.body
.replace(/^\s*<h1\b[^>]*>[\s\S]*?<\/h1>\s*/i, '')
.trim();
return sanitizeChapterHtml(withoutTitle);
return sanitizeChapterHtml(res.body.trim());

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Exported headings appear twice

The retained chapter heading reaches EPUB export unchanged. createEpub already adds its own heading before the returned content, so exported LNCrawler chapters now start with two headings. Keep the heading for reading, but make the export path avoid adding another when the content already has one.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Keeping the heading is intentional here: without it readers opened chapters with no title/number at all in the chapter header (this was reported by users reading on the app). The heading in the body faithfully mirrors what the source API provides for every chapter. The EPUB-export duplication is app-side — createEpub could skip adding its own title when the chapter content already starts with a heading (same guard other heading-preserving plugins would benefit from). Happy to revisit the opposite way if the app-side guard lands first.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense. The source heading is needed for the in-app reading experience, and the EPUB duplication is better handled by a generic guard in createEpub than by stripping or altering this plugin’s content. I’m withdrawing the finding; it shouldn’t block this PR.

Tip: You can customize Greptile's behavior for this repo with .greptile/rules.md and .greptile/config.json.

}

/**
* Languages the LNCrawler search API understands (from the site's own
* frontend), used for the Languages filter. Responses repeat the
* `languages` param, which the API aliases to `language`.
*/
const LANGUAGES: { label: string; value: string }[] = [
{ label: 'English', value: 'en' },
{ label: 'French', value: 'fr' },
{ label: 'Spanish', value: 'es' },
{ label: 'German', value: 'de' },
{ label: 'Italian', value: 'it' },
{ label: 'Japanese', value: 'ja' },
{ label: 'Korean', value: 'ko' },
{ label: 'Chinese', value: 'zh' },
{ label: 'Portuguese', value: 'pt' },
{ label: 'Russian', value: 'ru' },
{ label: 'Arabic', value: 'ar' },
{ label: 'Hindi', value: 'hi' },
{ label: 'Thai', value: 'th' },
{ label: 'Vietnamese', value: 'vi' },
{ label: 'Indonesian', value: 'id' },
{ label: 'Turkish', value: 'tr' },
{ label: 'Polish', value: 'pl' },
{ label: 'Dutch', value: 'nl' },
{ label: 'Swedish', value: 'sv' },
{ label: 'Danish', value: 'da' },
];

const SORT_OPTIONS: { label: string; value: string }[] = [
{ label: 'Default', value: '' },
{ label: 'Title', value: 'title' },
{ label: 'Rating', value: 'rating' },
{ label: 'Date added', value: 'date_added' },
{ label: 'All-time views', value: 'popularity' },
{ label: 'Trending', value: 'trending' },
{ label: 'Last updated', value: 'last_updated' },
];

const STATUS_OPTIONS: { label: string; value: string }[] = [
{ label: 'All', value: '' },
{ label: 'Ongoing', value: 'Ongoing' },
{ label: 'Completed', value: 'Completed' },
{ label: 'On Hiatus', value: 'On Hiatus' },
{ label: 'Cancelled', value: 'Cancelled' },
{ label: 'Unknown', value: 'Unknown' },
];

const ADULT_OPTIONS: { label: string; value: string }[] = [
{ label: 'Default', value: '' },
{ label: 'Hide adult content', value: 'hide' },
{ label: 'Show everything', value: 'show' },
{ label: 'Adult content only', value: 'only' },
];

class LNCrawler implements Plugin.PluginBase {
id = 'lncrawler';
name = 'LNCrawler';
icon = 'src/en/lncrawler/icon.png';
site = 'https://lncrawler.monster';
version = '1.0.1';
version = '1.1.1';

filters = {
languages: {
type: FilterTypes.CheckboxGroup,
label: 'Languages',
value: ['en'],
options: LANGUAGES,
},
status: {
type: FilterTypes.Picker,
label: 'Status',
value: '',
options: STATUS_OPTIONS,
Comment on lines +415 to +419

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Filters cannot be cleared individually

The four new pickers default to '', but none offers an option for that value. After choosing a status, readers cannot return just that field to “All” without resetting their other choices. The default sort and adult-content settings have the same problem. Add a labeled empty option to each picker so readers can clear each choice independently.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in a045b06: every picker (status/sort/order/adult) now starts with an empty-value option (All / Default) so any single filter can be reset without touching the others.

},
sort_by: {
type: FilterTypes.Picker,
label: 'Sort by',
value: '',
options: SORT_OPTIONS,
},
sort_order: {
type: FilterTypes.Picker,
label: 'Order',
value: '',
options: [
{ label: 'Default', value: '' },
{ label: 'Ascending', value: 'asc' },
{ label: 'Descending', value: 'desc' },
],
},
adult: {
type: FilterTypes.Picker,
label: 'Adult content',
value: '',
options: ADULT_OPTIONS,
},
} satisfies Filters;

/**
* Build the /novels/search/ query from filters. Catalog defaults:
* popular = all-time views descending; the Latest toggle switches to
* last-updated descending. An empty filter value means "server
* default" and is simply omitted.
*/
private buildSearchQuery(
options: Plugin.PopularNovelsOptions<typeof this.filters>,
pageNo: number,
query?: string,
): URLSearchParams {
const params = new URLSearchParams({
page: String(pageNo),
page_size: String(PAGE_SIZE),
});
if (query) params.set('query', query);
const opts: Plugin.PopularNovelsOptions<typeof this.filters> = options || {
showLatestNovels: false,
filters: undefined,
};
const filters = opts.filters;
if (!filters && !options) {
// Interface callers pass no options: catalog default = English.
params.append('languages', 'en');
} else {
for (const language of filters?.languages?.value || []) {
if (language) params.append('languages', language);
Comment on lines +470 to +471

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Chapters ignore the chosen language

Selecting a non-English language changes the catalog request, but opening a matching novel still calls chooseSource, which prefers English whenever English chapters exist. Cards carry only the novel slug, so the language choice never reaches the chapter source choice. Carry the selected language into the saved path and use it when choosing the source.

Knowledge Base Used: English source plugins

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in a045b06: the catalog now carries the selected languages in the novel path (slug?lang=a,b), and chooseSource prefers a source in one of those languages (first listed first) over the default English preference. Bare-slug/safe fallback (English-first) kept for paths saved before this change.

}
}
if (filters?.status?.value) params.set('status', filters.status.value);
if (filters?.sort_by?.value) {
params.set('sort_by', filters.sort_by.value);
} else {
params.set(
'sort_by',
opts.showLatestNovels ? 'last_updated' : 'popularity',
);
}
if (filters?.sort_order?.value) {
params.set('sort_order', filters.sort_order.value);
} else {
params.set('sort_order', 'desc');
}
if (filters?.adult?.value) params.set('adult', filters.adult.value);
return params;
}

async popularNovels(pageNo: number): Promise<Plugin.NovelItem[]> {
async popularNovels(
pageNo: number,
options: Plugin.PopularNovelsOptions<typeof this.filters>,
): Promise<Plugin.NovelItem[]> {
// The search endpoint is the only listing with a language filter;
// with no query it serves as the catalog. The API exposes no sort
// options, so the Latest toggle lands here too.
// with no query it serves as the catalog. Popular = all-time views
// descending, the Latest toggle = last-updated descending.
const data = await fetchJson(
`${API}/novels/search/?language=en&page=${pageNo}&page_size=${PAGE_SIZE}`,
`${API}/novels/search/?${this.buildSearchQuery(options, pageNo)}`,
'Failed to load popular novels',
);
const page = parseNovelListPage(data);
return pageNo > page.totalPages ? [] : page.cards;
return this.tagWithLanguages(page, pageNo, options);
}

/** Selected languages carried in the novel path, if any (`slug?lang=a,b`). */
private langsFromPath(novelPath: string): string[] {
const url = new URL(novelPath, this.site);
const raw = url.searchParams.get('lang') || '';
return raw.split(',').filter(Boolean);
}

async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
const slug = novelPath.replace(/^\/novels\//, '').replace(/\/+$/, '');
// The path may be a bare slug (`shadow-slave?lang=en`) or a resolved
// site URL (`.../novels/shadow-slave?lang=en`); the last path
// segment is the novel slug either way.
const slug =
new URL(novelPath, this.site).pathname.split('/').filter(Boolean).pop() ||
'';
const detail = parseNovelSources(
await fetchJson(
`${API}/novels/${encodeURIComponent(slug)}/`,
Expand All @@ -347,7 +527,7 @@ class LNCrawler implements Plugin.PluginBase {
if (!detail.name) {
throw new Error('Could not load the novel ' + novelPath);
}
const source = chooseSource(detail);
const source = chooseSource(detail, this.langsFromPath(novelPath));
const base =
`${API}/novels/${encodeURIComponent(slug)}` +
`/${encodeURIComponent(source.slug)}`;
Expand Down Expand Up @@ -417,13 +597,33 @@ class LNCrawler implements Plugin.PluginBase {
async searchNovels(
searchTerm: string,
pageNo: number,
options?: Plugin.PopularNovelsOptions<typeof this.filters>,
): Promise<Plugin.NovelItem[]> {
const data = await fetchJson(
`${API}/novels/search/?query=${encodeURIComponent(searchTerm)}&language=en&page=${pageNo}&page_size=${PAGE_SIZE}`,
`${API}/novels/search/?${this.buildSearchQuery(options, pageNo, searchTerm)}`,
'Failed to search novels',
);
const page = parseNovelListPage(data);
return pageNo > page.totalPages ? [] : page.cards;
return this.tagWithLanguages(page, pageNo, options);
}

/**
* Catalog cards carry the chosen languages in their path
* (`slug?lang=a,b`) so parseNovel can pick a chapter source that
* matches them; used only when a language filter is actually set.
*/
private tagWithLanguages(
page: ListPage,
pageNo: number,
options?: Plugin.PopularNovelsOptions<typeof this.filters>,
): Plugin.NovelItem[] {
if (pageNo > page.totalPages) return [];
const langs = (options?.filters?.languages?.value || [])
.filter(Boolean)
.join(',');
if (!langs) return page.cards;
const suffix = `?lang=${encodeURIComponent(langs)}`;
return page.cards.map(card => ({ ...card, path: card.path + suffix }));
}

resolveUrl = (path: string) => {
Expand Down
Loading