diff --git a/lib/api/routes/userSettingsRoute.js b/lib/api/routes/userSettingsRoute.js index 78631266..9cbbb48c 100644 --- a/lib/api/routes/userSettingsRoute.js +++ b/lib/api/routes/userSettingsRoute.js @@ -118,4 +118,25 @@ userSettingsRouter.post('/immoscout-details', async (req, res) => { } }); +userSettingsRouter.post('/kleinanzeigen-details', async (req, res) => { + const userId = req.session.currentUser; + const { kleinanzeigen_details } = req.body; + + const globalSettings = await getSettings(); + if (globalSettings.demoMode) { + res.statusCode = 403; + res.send({ error: 'In demo mode, it is not allowed to change settings.' }); + return; + } + + try { + upsertSettings({ kleinanzeigen_details: !!kleinanzeigen_details }, userId); + res.send({ success: true }); + } catch (error) { + logger.error('Error updating kleinanzeigen details setting', error); + res.statusCode = 500; + res.send({ error: error.message }); + } +}); + export { userSettingsRouter }; diff --git a/lib/provider/kleinanzeigen.js b/lib/provider/kleinanzeigen.js index d72c474c..b32c4e09 100755 --- a/lib/provider/kleinanzeigen.js +++ b/lib/provider/kleinanzeigen.js @@ -5,15 +5,185 @@ import { buildHash, isOneOf } from '../utils.js'; import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; +import Extractor from '../services/extractor/extractor.js'; +import logger from '../services/logger.js'; +import { getUserSettings } from '../services/storage/settingsStorage.js'; +import * as cheerio from 'cheerio'; let appliedBlackList = []; let appliedBlacklistedDistricts = []; +let currentUserId = null; + +function toAbsoluteLink(link) { + if (!link) return null; + return link.startsWith('http') ? link : `https://www.kleinanzeigen.de${link}`; +} + +function cleanText(value) { + if (value == null) return ''; + return String(value) + .replace(/<[^>]*>/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function buildAddressFromJsonLd(address) { + if (!address || typeof address !== 'object') return null; + + const locality = cleanText(address.addressLocality); + const region = cleanText(address.addressRegion); + const postalCode = cleanText(address.postalCode); + const streetAddress = cleanText(address.streetAddress); + + const cityPart = [region, locality].filter(Boolean).join(' - '); + const tail = [postalCode, cityPart || locality || region].filter(Boolean).join(' '); + const fullAddress = [streetAddress, tail].filter(Boolean).join(', '); + + return fullAddress || null; +} + +function flattenJsonLdNodes(node, acc = []) { + if (node == null) return acc; + + if (Array.isArray(node)) { + node.forEach((item) => flattenJsonLdNodes(item, acc)); + return acc; + } + + if (typeof node !== 'object') return acc; + + acc.push(node); + + if (Array.isArray(node['@graph'])) { + node['@graph'].forEach((item) => flattenJsonLdNodes(item, acc)); + } + + if (node.mainEntity) { + flattenJsonLdNodes(node.mainEntity, acc); + } + + if (node.itemOffered) { + flattenJsonLdNodes(node.itemOffered, acc); + } + + return acc; +} + +function extractDetailFromHtml(html) { + const $ = cheerio.load(html); + const nodes = []; + + // Prefer the rendered postal address block from the detail page because + // it contains the street line that is missing from list results. + const streetFromDom = cleanText($('#street-address').first().text()); + const localityFromDom = cleanText($('#viewad-locality').first().text()); + const domAddress = [streetFromDom, localityFromDom].filter(Boolean).join(' '); + + $('script[type="application/ld+json"]').each((_, element) => { + const content = $(element).text(); + if (!content) return; + + try { + const parsed = JSON.parse(content); + flattenJsonLdNodes(parsed, nodes); + } catch { + // Ignore broken JSON-LD blocks from ads/trackers and keep trying others. + } + }); + + let detailAddress = null; + let detailDescription = null; + + if (domAddress) { + detailAddress = domAddress; + } + + for (const node of nodes) { + const candidateAddress = buildAddressFromJsonLd( + node.address || node?.itemOffered?.address || node?.offers?.address, + ); + if (!detailAddress && candidateAddress) { + detailAddress = candidateAddress; + } + + const candidateDescription = cleanText(node.description || node?.itemOffered?.description); + if (!detailDescription && candidateDescription) { + detailDescription = candidateDescription; + } + + if (detailAddress && detailDescription) { + break; + } + } + + return { + detailAddress, + detailDescription, + }; +} + +async function enrichListingFromDetails(listing) { + const absoluteLink = toAbsoluteLink(listing.link); + if (!absoluteLink) return listing; + + try { + const response = await fetch(absoluteLink, { + headers: { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36', + }, + }); + + if (!response.ok) { + return { + ...listing, + link: absoluteLink, + }; + } + + const html = await response.text(); + const { detailAddress, detailDescription } = extractDetailFromHtml(html); + + return { + ...listing, + link: absoluteLink, + address: detailAddress || listing.address, + description: detailDescription || listing.description, + }; + } catch (error) { + logger.warn(`Could not fetch Kleinanzeigen detail page for listing '${listing.id}'.`, error?.message || error); + return { + ...listing, + link: absoluteLink, + }; + } +} + +async function getListings(url) { + const extractor = new Extractor(); + await extractor.execute(url, config.waitForSelector); + const listings = extractor.parseResponseText(config.crawlContainer, config.crawlFields, url) || []; + + const shouldFetchDetails = currentUserId ? !!getUserSettings(currentUserId).kleinanzeigen_details : false; + + if (!shouldFetchDetails) { + return listings.map((listing) => ({ + ...listing, + link: toAbsoluteLink(listing.link) || listing.link, + })); + } + + const enriched = []; + for (const listing of listings) { + enriched.push(await enrichListingFromDetails(listing)); + } + return enriched; +} function normalize(o) { const size = o.size || '--- m²'; const id = buildHash(o.id, o.price); - const link = `https://www.kleinanzeigen.de${o.link}`; - return Object.assign(o, { id, size, link }); + return Object.assign(o, { id, size }); } function applyBlacklist(o) { @@ -40,12 +210,13 @@ const config = { address: '.aditem-main--top--left | trim | removeNewline', image: 'img@src', }, + getListings: getListings, normalize: normalize, filter: applyBlacklist, activeTester: checkIfListingIsActive, }; export const metaInformation = { - name: 'Ebay Kleinanzeigen', + name: 'Kleinanzeigen', baseUrl: 'https://www.kleinanzeigen.de/', id: 'kleinanzeigen', }; @@ -54,5 +225,6 @@ export const init = (sourceConfig, blacklist, blacklistedDistricts) => { config.url = sourceConfig.url; appliedBlacklistedDistricts = blacklistedDistricts || []; appliedBlackList = blacklist || []; + currentUserId = sourceConfig.userId || null; }; export { config }; diff --git a/ui/src/services/state/store.js b/ui/src/services/state/store.js index d76f1dd4..c3718d32 100644 --- a/ui/src/services/state/store.js +++ b/ui/src/services/state/store.js @@ -318,6 +318,20 @@ export const useFredyState = create( throw Exception; } }, + async setKleinanzeigenDetails(enabled) { + try { + await xhrPost('/api/user/settings/kleinanzeigen-details', { kleinanzeigen_details: enabled }); + set((state) => ({ + userSettings: { + ...state.userSettings, + settings: { ...state.userSettings.settings, kleinanzeigen_details: enabled }, + }, + })); + } catch (Exception) { + console.error('Error while trying to update kleinanzeigen details setting. Error:', Exception); + throw Exception; + } + }, }, }; diff --git a/ui/src/views/generalSettings/GeneralSettings.jsx b/ui/src/views/generalSettings/GeneralSettings.jsx index d0da96a2..b6c3ec8f 100644 --- a/ui/src/views/generalSettings/GeneralSettings.jsx +++ b/ui/src/views/generalSettings/GeneralSettings.jsx @@ -73,6 +73,7 @@ const GeneralSettings = function GeneralSettings() { // User settings state const homeAddress = useSelector((state) => state.userSettings.settings.home_address); const immoscoutDetails = useSelector((state) => state.userSettings.settings.immoscout_details); + const kleinanzeigenDetails = useSelector((state) => state.userSettings.settings.kleinanzeigen_details); const [address, setAddress] = useState(homeAddress?.address || ''); const [coords, setCoords] = useState(homeAddress?.coords || null); const saving = useIsLoading(actions.userSettings.setHomeAddress); @@ -465,6 +466,32 @@ const GeneralSettings = function GeneralSettings() { + + +
+ { + try { + await actions.userSettings.setKleinanzeigenDetails(checked); + Toast.success('Kleinanzeigen details setting updated.'); + } catch { + Toast.error('Failed to update setting.'); + } + }} + /> + Fetch detailed Kleinanzeigen listings +
+ +
+ + +
+ { + try { + await actions.userSettings.setKleinanzeigenDetails(checked); + Toast.success('Kleinanzeigen details setting updated.'); + } catch { + Toast.error('Failed to update setting.'); + } + }} + /> + Fetch detailed Kleinanzeigen listings +
+ +