s.yahyaee/car-crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First version of new crawler

Browse Source
This commit is contained in:
SepehrYahyaee
2026-03-14 14:07:52 +03:30
parent abc0bd9bee
commit c9cf830d7b
35 changed files with 27108 additions and 1174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
.gitignore vendored
View File

@@ -1,58 +1,146 @@
### Node ###
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (http://nodejs.org/api/addons.html)
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules
jspm_packages
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# 0x
profile-*
# Output of 'npm pack'
*.tgz
# mac files
.DS_Store
# Yarn Integrity file
.yarn-integrity
# vim swap files
*.swp
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
.env.development.env
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# webstorm
.idea
# Next.js build output
.next
out
# vscode
.vscode
*code-workspace
# Nuxt.js build / generate output
.nuxt
dist
# clinic
profile*
*clinic*
*flamegraph*
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*
### Node Patch ###
# Serverless Webpack directories
.webpack/
# Optional stylelint cache
# SvelteKit build / generate output
.svelte-kit
# End of https://www.toptal.com/developers/gitignore/api/node
/docker
.development.env
*.env
/files

23
README.md
View File

@@ -1,23 +0,0 @@
# Getting Started with [Fastify-CLI](https://www.npmjs.com/package/fastify-cli)
This project was bootstrapped with Fastify-CLI.
## Available Scripts
In the project directory, you can run:
### `npm run dev`
To start the app in dev mode.\
Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
### `npm start`
For production mode
### `npm run test`
Run the test cases.
## Learn More
To learn Fastify, check out the [Fastify documentation](https://fastify.dev/docs/latest/).

30
app.js
View File

@@ -1,30 +0,0 @@
'use strict'
const path = require('node:path')
const AutoLoad = require('@fastify/autoload')
// Pass --options via CLI arguments in command to enable these options.
const options = {}
module.exports = async function (fastify, opts) {
// Place here your custom code!
// Do not touch the following lines
// This loads all plugins defined in plugins
// those should be support plugins that are reused
// through your application
fastify.register(AutoLoad, {
dir: path.join(__dirname, 'plugins'),
options: Object.assign({}, opts)
})
// This loads all plugins defined in routes
// define your routes in one of these
fastify.register(AutoLoad, {
dir: path.join(__dirname, 'routes'),
options: Object.assign({}, opts)
})
}
module.exports.options = options

1
json/akharin/2026-1-7-akharin-cars.json Normal file
View File

@@ -0,0 +1 @@
[]

2167
json/hamrah/2026-1-7-hamrah-cars.json Normal file
View File

File diff suppressed because it is too large Load Diff

19475
json/yadakmarket/2026-1-7-yadakmarket-parts.json Normal file
View File

File diff suppressed because it is too large Load Diff

8
nest-cli.json Normal file
View File

@@ -0,0 +1,8 @@
{
"$schema": "https://json.schemastore.org/nest-cli",
"collection": "@nestjs/schematics",
"sourceRoot": "src",
"compilerOptions": {
"deleteOutDir": true
}
}

5337
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

38
package.json
View File

@@ -2,25 +2,43 @@
"name": "myproject",
"version": "1.0.0",
"description": "This project was bootstrapped with Fastify-CLI.",
"main": "app.js",
"main": "dist/main.js",
"directories": {
"test": "test"
},
"scripts": {
"build": "tsc -p tsconfig.build.json",
"start": "node dist/main.js",
"start:dev": "ts-node-dev --respawn --transpile-only -r tsconfig-paths/register src/main.ts",
"start:prod": "node dist/main.js",
"test": "node --test test/**/*.test.js",
"start": "fastify start -l info app.js",
"dev": "fastify start -w -l info -P app.js"
"crawl": "ts-node -r tsconfig-paths/register src/cli/crawl.ts"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@fastify/autoload": "^5.0.0",
"@fastify/sensible": "^5.0.0",
"axios": "^1.9.0",
"fastify": "^4.26.1",
"fastify-cli": "^6.1.1",
"fastify-cron": "^1.4.0",
"fastify-plugin": "^4.0.0"
"@nestjs/common": "^11.1.12",
"@nestjs/config": "^4.0.2",
"@nestjs/core": "^11.1.12",
"@nestjs/mongoose": "^11.0.4",
"@nestjs/platform-express": "^11.1.12",
"@nestjs/schedule": "^6.1.0",
"@nestjs/swagger": "^11.2.5",
"axios": "^1.13.2",
"cheerio": "^1.2.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.3",
"mongoose": "^9.1.5",
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.2"
},
"devDependencies": {
"@nestjs/cli": "^11.0.16",
"@types/node": "^25.0.10",
"ts-node": "^10.9.2",
"ts-node-dev": "^2.0.0",
"tsconfig-paths": "^4.2.0",
"typescript": "^5.9.3"
}
}

16
plugins/README.md
View File

@@ -1,16 +0,0 @@
# Plugins Folder
Plugins define behavior that is common to all the routes in your
application. Authentication, caching, templates, and all the other cross
cutting concerns should be handled by plugins placed in this folder.
Files in this folder are typically defined through the
[`fastify-plugin`](https://github.com/fastify/fastify-plugin) module,
making them non-encapsulated. They can define decorators and set hooks
that will then be used in the rest of your application.
Check out:
* [The hitchhiker's guide to plugins](https://fastify.dev/docs/latest/Guides/Plugins-Guide/)
* [Fastify decorators](https://fastify.dev/docs/latest/Reference/Decorators/).
* [Fastify lifecycle](https://fastify.dev/docs/latest/Reference/Lifecycle/).

96
plugins/cron.js
View File

@@ -1,96 +0,0 @@
'use strict'
const fp = require('fastify-plugin')
const fastifyCron = require('fastify-cron')
const cheerio = require('cheerio');
const { log } = require('console');
const fs = require('fs');
const axios = require('axios');
const akharin = 'https://akharinkhodro.ir/price/%D9%82%DB%8C%D9%85%D8%AA-%D8%AE%D9%88%D8%AF%D8%B1%D9%88/';
const hamrah = 'https://www.hamrah-mechanic.com/carprice/';
module.exports = fp(async function (fastify, opts) {
fastify.register(fastifyCron, {
jobs: [
{
cronTime: '0 0 * * *', ///'*/5 * * * * *'
onTick: async () => {
fastify.log.info('⏰ Cron Job ran at ' + new Date().toISOString())
try {
const { data } = await axios.get(akharin, {
headers: { 'User-Agent': 'Mozilla/5.0' },
});
const $ = cheerio.load(data);
const cars = [];
let td = $('tr')
let result = []
td.each((i, tr) => {
let listOfLine = $(tr).text().trim().split('\n')
if (listOfLine.length > 1) {
if (listOfLine[0] !== "نام خودرو") {
result.push({
carName: listOfLine[0],
marketPrice: listOfLine[1].trim().split(" ")[0] === "بدون" ? null : listOfLine[1].trim().split(" ")[0],
newCarPrice: listOfLine[2].trim().split(" ")[0]
})
}
}
})
fs.writeFileSync(`./json/akharin/${new Date().getFullYear()}-${new Date().getMonth()}-${new Date().getDay()}-akharin-cars.json`, JSON.stringify(result, null, 2), 'utf-8');
console.log(`${new Date().getFullYear()}-$${new Date().getMonth()}-${new Date().getDay()}-akharin-cars.json => new:`, cars.length);
} catch (err) {
console.error('crawler stop:', err.message);
} },
start: true
},{
cronTime: '0 0 * * *', ///'*/5 * * * * *'
onTick: async () => {
fastify.log.info('⏰ Cron Job ran at hamrah ' + new Date().toISOString())
try {
const { data } = await axios.get(hamrah, {
headers: { 'User-Agent': 'Mozilla/5.0' },
});
const $ = cheerio.load(data);
const cars = [];
const result = []
let sectionName = $('div.carsBrandPriceList_brand__name__Ohntn')
let tables = $('tbody tr');
tables.each((i, tr) => {
const tds = $(tr).find('td');
const modelDivs = $(tds[0]).find('div');
const priceDivs = $(tds[1]).find('div');
const carName = $(modelDivs[0]).text().trim(); // مدل اصلی مثل Q5 E-tron
const typeAndYear = $(modelDivs[1]).text().replace(/\s+/g, ' ').trim(); // تیپ 40- فول-2024
const marketPrice = $(priceDivs[0]).text().trim(); // قیمت
result.push({carName , typeAndYear , marketPrice})
});
// ذخیره‌سازی در فایل JSON
fs.writeFileSync(`./json/hamrah/${new Date().getFullYear()}-${new Date().getMonth()}-${new Date().getDay()}-hamrah-cars.json`, JSON.stringify(result, null, 2), 'utf-8');
console.log(`${new Date().getFullYear()}-$${new Date().getMonth()}-${new Date().getDay()}-akharin-cars.json => new:`, cars.length);
} catch (err) {
console.error('❌ خطا در کراول:', err.message);
} },
start: true
}
]
})
})

14
plugins/sensible.js
View File

@@ -1,14 +0,0 @@
'use strict'
const fp = require('fastify-plugin')
/**
* This plugins adds some utilities to handle http errors
*
* @see https://github.com/fastify/fastify-sensible
*/
module.exports = fp(async function (fastify, opts) {
fastify.register(require('@fastify/sensible'), {
errorHandler: false
})
})

12
plugins/support.js
View File

@@ -1,12 +0,0 @@
'use strict'
const fp = require('fastify-plugin')
// the use of fastify-plugin is required to be able
// to export the decorators to the outer scope
module.exports = fp(async function (fastify, opts) {
fastify.decorate('someSupport', function () {
return 'hugs'
})
})

27
routes/README.md
View File

@@ -1,27 +0,0 @@
# Routes Folder
Routes define routes within your application. Fastify provides an
easy path to a microservice architecture, in the future you might want
to independently deploy some of those.
In this folder you should define all the routes that define the endpoints
of your web application.
Each service is a [Fastify
plugin](https://fastify.dev/docs/latest/Reference/Plugins/), it is
encapsulated (it can have its own independent plugins) and it is
typically stored in a file; be careful to group your routes logically,
e.g. all `/users` routes in a `users.js` file. We have added
a `root.js` file for you with a '/' root added.
If a single file become too large, create a folder and add a `index.js` file there:
this file must be a Fastify plugin, and it will be loaded automatically
by the application. You can now add as many files as you want inside that folder.
In this way you can create complex routes within a single monolith,
and eventually extract them.
If you need to share functionality between routes, place that
functionality into the `plugins` folder, and share it via
[decorators](https://fastify.dev/docs/latest/Reference/Decorators/).
If you're a bit confused about using `async/await` to write routes, you would
better take a look at [Promise resolution](https://fastify.dev/docs/latest/Reference/Routes/#promise-resolution) for more details.

7
routes/example/index.js
View File

@@ -1,7 +0,0 @@
'use strict'
module.exports = async function (fastify, opts) {
fastify.get('/', async function (request, reply) {
return 'this is an example'
})
}

15
routes/root.js
View File

@@ -1,15 +0,0 @@
'use strict'
const { log } = require("console")
const {readFile , readdir} = require('fs/promises')
module.exports = async function (fastify, opts) {
fastify.get('/price', async function (request, reply) {
if(Object.keys(request.query).length == 0) return reply.status(500).send({ ok: false, message:'use query params' , enums:['akharin' , 'hamrah'] })
let dir = await readdir(`${process.cwd()}/json/${Object.keys(request.query)[0]}`)
let file = await readFile(`${process.cwd()}/json/${Object.keys(request.query)[0]}/${dir[0]}` , 'utf-8')
return JSON.parse(file)
})
}

52
src/app.module.ts Normal file
View File

@@ -0,0 +1,52 @@
import { Module } from '@nestjs/common';
import { ConfigModule, ConfigService } from '@nestjs/config';
import { MongooseModule } from '@nestjs/mongoose';
import { ScheduleModule } from '@nestjs/schedule';
import { CrawlerService } from './crawler/crawler.service';
import { PricesController } from './controllers/prices.controller';
import { SearchController } from './controllers/search.controller';
import { StatsController } from './controllers/stats.controller';
import { HealthController } from './controllers/health.controller';
import { YadakmarketController } from './controllers/yadakmarket.controller';
import { CrawlController } from './controllers/crawl.controller';
import { AkharinCar, AkharinCarSchema } from './schemas/akharin-car.schema';
import { HamrahCar, HamrahCarSchema } from './schemas/hamrah-car.schema';
import { YadakmarketPart, YadakmarketPartSchema } from './schemas/yadakmarket-part.schema';
@Module({
imports: [
ConfigModule.forRoot({
isGlobal: true,
envFilePath: '.env',
expandVariables: true,
}),
ScheduleModule.forRoot(),
MongooseModule.forRootAsync({
useFactory: (config: ConfigService) => {
const uri =
config.get<string>('CAR_CRAWLER_DB_CONNECTION_STRING') ||
process.env.CAR_CRAWLER_DB_CONNECTION_STRING;
if (!uri || uri.includes('${')) {
throw new Error('CAR_CRAWLER_DB_CONNECTION_STRING is required and must be a valid MongoDB URI.');
}
return { uri };
},
inject: [ConfigService],
}),
MongooseModule.forFeature([
{ name: AkharinCar.name, schema: AkharinCarSchema },
{ name: HamrahCar.name, schema: HamrahCarSchema },
{ name: YadakmarketPart.name, schema: YadakmarketPartSchema },
]),
],
controllers: [
PricesController,
SearchController,
StatsController,
HealthController,
YadakmarketController,
CrawlController,
],
providers: [CrawlerService],
})
export class AppModule {}

16
src/cli/crawl.ts Normal file
View File

@@ -0,0 +1,16 @@
import { NestFactory } from '@nestjs/core';
import { AppModule } from '../app.module';
import { CrawlerService } from '../crawler/crawler.service';
async function run() {
const app = await NestFactory.createApplicationContext(AppModule);
const crawler = app.get(CrawlerService);
await crawler.runAll();
await app.close();
}
run().catch((err) => {
// eslint-disable-next-line no-console
console.error(err);
process.exit(1);
});

16
src/controllers/crawl.controller.ts Normal file
View File

@@ -0,0 +1,16 @@
import { Controller, Post } from '@nestjs/common';
import { ApiTags, ApiResponse } from '@nestjs/swagger';
import { CrawlerService } from '../crawler/crawler.service';
@ApiTags('crawler')
@Controller()
export class CrawlController {
constructor(private readonly crawler: CrawlerService) {}
@Post('crawl')
@ApiResponse({ status: 200, description: 'Trigger a crawl immediately.' })
async runCrawl() {
await this.crawler.runAll();
return { ok: true, message: 'Crawl completed' };
}
}

15
src/controllers/health.controller.ts Normal file
View File

@@ -0,0 +1,15 @@
import { Controller, Get } from '@nestjs/common';
import { ApiTags, ApiResponse } from '@nestjs/swagger';
@ApiTags('health')
@Controller()
export class HealthController {
@Get('health')
@ApiResponse({ status: 200, description: 'Health check.' })
health() {
return {
status: 'ok',
timestamp: new Date().toISOString(),
};
}
}

68
src/controllers/prices.controller.ts Normal file
View File

@@ -0,0 +1,68 @@
import { Controller, Get, Query, BadRequestException } from '@nestjs/common';
import { ApiQuery, ApiTags, ApiResponse } from '@nestjs/swagger';
import { InjectModel } from '@nestjs/mongoose';
import { Model } from 'mongoose';
import { AkharinCar, AkharinCarDocument } from '../schemas/akharin-car.schema';
import { HamrahCar, HamrahCarDocument } from '../schemas/hamrah-car.schema';
import { YadakmarketPart, YadakmarketPartDocument } from '../schemas/yadakmarket-part.schema';
@ApiTags('prices')
@Controller()
export class PricesController {
constructor(
@InjectModel(AkharinCar.name) private readonly akharinModel: Model<AkharinCarDocument>,
@InjectModel(HamrahCar.name) private readonly hamrahModel: Model<HamrahCarDocument>,
@InjectModel(YadakmarketPart.name) private readonly yadakModel: Model<YadakmarketPartDocument>,
) {}
@Get('price')
@ApiQuery({ name: 'source', enum: ['akharin', 'hamrah', 'yadakmarket'], required: true })
@ApiResponse({ status: 200, description: 'List of prices by source.' })
async getPrices(@Query('source') source?: string) {
if (!source) {
throw new BadRequestException({
ok: false,
message: 'use query params',
enums: ['akharin', 'hamrah', 'yadakmarket'],
example: '/price?source=akharin',
});
}
switch (source) {
case 'akharin': {
const data = await this.akharinModel.find({}).sort({ crawledAt: -1 }).limit(1000).lean();
return data.map((doc) => ({
carName: doc.carName,
marketPrice: doc.marketPrice,
newCarPrice: doc.newCarPrice,
}));
}
case 'hamrah': {
const data = await this.hamrahModel.find({}).sort({ crawledAt: -1 }).limit(1000).lean();
return data.map((doc) => ({
carName: doc.carName,
typeAndYear: doc.typeAndYear,
marketPrice: doc.marketPrice,
}));
}
case 'yadakmarket': {
const data = await this.yadakModel.find({}).sort({ crawledAt: -1 }).limit(1000).lean();
return data
.filter((doc) => doc && doc.productName)
.map((doc) => ({
productName: doc.productName,
productUrl: doc.productUrl || null,
productId: doc.productId || null,
currentPrice: doc.currentPrice ?? null,
maxPrice: doc.maxPrice ?? null,
}));
}
default:
throw new BadRequestException({
ok: false,
message: 'Invalid source',
enums: ['akharin', 'hamrah', 'yadakmarket'],
});
}
}
}

82
src/controllers/search.controller.ts Normal file
View File

@@ -0,0 +1,82 @@
import { Controller, Get, Query, BadRequestException } from '@nestjs/common';
import { ApiQuery, ApiTags, ApiResponse } from '@nestjs/swagger';
import { InjectModel } from '@nestjs/mongoose';
import { Model } from 'mongoose';
import { AkharinCar, AkharinCarDocument } from '../schemas/akharin-car.schema';
import { HamrahCar, HamrahCarDocument } from '../schemas/hamrah-car.schema';
import { YadakmarketPart, YadakmarketPartDocument } from '../schemas/yadakmarket-part.schema';
@ApiTags('search')
@Controller()
export class SearchController {
constructor(
@InjectModel(AkharinCar.name) private readonly akharinModel: Model<AkharinCarDocument>,
@InjectModel(HamrahCar.name) private readonly hamrahModel: Model<HamrahCarDocument>,
@InjectModel(YadakmarketPart.name) private readonly yadakModel: Model<YadakmarketPartDocument>,
) {}
@Get('search')
@ApiQuery({ name: 'q', required: true, type: String })
@ApiQuery({ name: 'source', required: false, enum: ['akharin', 'hamrah', 'yadakmarket'] })
@ApiResponse({ status: 200, description: 'Search results across sources.' })
async search(@Query('q') q?: string, @Query('source') source?: string) {
if (!q) {
throw new BadRequestException({
ok: false,
message: 'Query parameter "q" is required',
example: '/search?q=پژو&source=akharin',
});
}
const searchRegex = new RegExp(q, 'i');
const results: any[] = [];
if (source === 'akharin' || !source) {
const data = await this.akharinModel.find({ carName: searchRegex }).limit(100).lean();
results.push(
...data.map((doc) => ({
source: 'akharin',
carName: doc.carName,
marketPrice: doc.marketPrice,
newCarPrice: doc.newCarPrice,
})),
);
}
if (source === 'hamrah' || !source) {
const data = await this.hamrahModel.find({ carName: searchRegex }).limit(100).lean();
results.push(
...data.map((doc) => ({
source: 'hamrah',
carName: doc.carName,
typeAndYear: doc.typeAndYear,
marketPrice: doc.marketPrice,
})),
);
}
if (source === 'yadakmarket' || !source) {
const data = await this.yadakModel.find({ productName: searchRegex }).limit(100).lean();
results.push(
...data
.filter((doc) => doc && doc.productName)
.map((doc) => ({
source: 'yadakmarket',
productName: doc.productName,
productUrl: doc.productUrl || null,
productId: doc.productId || null,
currentPrice: doc.currentPrice ?? null,
maxPrice: doc.maxPrice ?? null,
})),
);
}
return {
ok: true,
query: q,
source: source || 'all',
count: results.length,
results,
};
}
}

51
src/controllers/stats.controller.ts Normal file
View File

@@ -0,0 +1,51 @@
import { Controller, Get } from '@nestjs/common';
import { ApiTags, ApiResponse } from '@nestjs/swagger';
import { InjectModel } from '@nestjs/mongoose';
import { Model } from 'mongoose';
import { AkharinCar, AkharinCarDocument } from '../schemas/akharin-car.schema';
import { HamrahCar, HamrahCarDocument } from '../schemas/hamrah-car.schema';
import { YadakmarketPart, YadakmarketPartDocument } from '../schemas/yadakmarket-part.schema';
@ApiTags('prices')
@Controller()
export class StatsController {
constructor(
@InjectModel(AkharinCar.name) private readonly akharinModel: Model<AkharinCarDocument>,
@InjectModel(HamrahCar.name) private readonly hamrahModel: Model<HamrahCarDocument>,
@InjectModel(YadakmarketPart.name) private readonly yadakModel: Model<YadakmarketPartDocument>,
) {}
@Get('stats')
@ApiResponse({ status: 200, description: 'Stats for each source.' })
async getStats() {
const [akharinCount, akharinLast] = await Promise.all([
this.akharinModel.countDocuments(),
this.akharinModel.findOne().sort({ crawledAt: -1 }).select('crawledAt').lean(),
]);
const [hamrahCount, hamrahLast] = await Promise.all([
this.hamrahModel.countDocuments(),
this.hamrahModel.findOne().sort({ crawledAt: -1 }).select('crawledAt').lean(),
]);
const [yadakCount, yadakLast] = await Promise.all([
this.yadakModel.countDocuments(),
this.yadakModel.findOne().sort({ crawledAt: -1 }).select('crawledAt').lean(),
]);
return {
akharin: {
total: akharinCount,
lastCrawled: akharinLast?.crawledAt?.toISOString() || null,
},
hamrah: {
total: hamrahCount,
lastCrawled: hamrahLast?.crawledAt?.toISOString() || null,
},
yadakmarket: {
total: yadakCount,
lastCrawled: yadakLast?.crawledAt?.toISOString() || null,
},
};
}
}

31
src/controllers/yadakmarket.controller.ts Normal file
View File

@@ -0,0 +1,31 @@
import { Controller, Get, Param, NotFoundException } from '@nestjs/common';
import { ApiTags, ApiResponse, ApiParam } from '@nestjs/swagger';
import { InjectModel } from '@nestjs/mongoose';
import { Model } from 'mongoose';
import { YadakmarketPart, YadakmarketPartDocument } from '../schemas/yadakmarket-part.schema';
@ApiTags('prices')
@Controller()
export class YadakmarketController {
constructor(
@InjectModel(YadakmarketPart.name) private readonly yadakModel: Model<YadakmarketPartDocument>,
) {}
@Get('yadakmarket/:productId')
@ApiParam({ name: 'productId', type: String })
@ApiResponse({ status: 200, description: 'Single yadakmarket product.' })
async getById(@Param('productId') productId: string) {
const product = await this.yadakModel.findOne({ productId }).lean();
if (!product) {
throw new NotFoundException({ ok: false, message: 'Product not found' });
}
return {
productName: product.productName,
productUrl: product.productUrl || null,
productId: product.productId || null,
currentPrice: product.currentPrice ?? null,
maxPrice: product.maxPrice ?? null,
};
}
}

284
src/crawler/crawler.service.ts Normal file
View File

@@ -0,0 +1,284 @@
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { InjectModel } from '@nestjs/mongoose';
import { Model } from 'mongoose';
import axios from 'axios';
import * as cheerio from 'cheerio';
import { AkharinCar, AkharinCarDocument } from '../schemas/akharin-car.schema';
import { HamrahCar, HamrahCarDocument } from '../schemas/hamrah-car.schema';
import { YadakmarketPart, YadakmarketPartDocument } from '../schemas/yadakmarket-part.schema';
@Injectable()
export class CrawlerService {
private readonly logger = new Logger(CrawlerService.name);
private readonly akharinUrl = 'https://akharinkhodro.ir/price/%D9%82%DB%8C%D9%85%D8%AA-%D8%AE%D9%88%D8%AF%D8%B1%D9%88/';
private readonly hamrahUrl = 'https://www.hamrah-mechanic.com/carprice/';
constructor(
@InjectModel(AkharinCar.name) private readonly akharinModel: Model<AkharinCarDocument>,
@InjectModel(HamrahCar.name) private readonly hamrahModel: Model<HamrahCarDocument>,
@InjectModel(YadakmarketPart.name) private readonly yadakModel: Model<YadakmarketPartDocument>,
) {}
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async cronAkharin(): Promise<void> {
await this.crawlAkharin();
}
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async cronHamrah(): Promise<void> {
await this.crawlHamrah();
}
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async cronYadakmarket(): Promise<void> {
await this.crawlYadakmarket();
}
async runAll(): Promise<void> {
await this.crawlAkharin();
await this.crawlHamrah();
await this.crawlYadakmarket();
}
private parseYadakPrice(
priceHtml: string | null,
): { currentPrice: string | null; maxPrice: string | null } {
if (!priceHtml) return { currentPrice: null, maxPrice: null };
const $ = cheerio.load(priceHtml);
if ($('a[href^="tel:"]').length > 0) return { currentPrice: null, maxPrice: null };
const priceAmounts: string[] = [];
$('.woocommerce-Price-amount').each((_, el) => {
const priceText = $(el).find('bdi').text().trim();
const cleaned = priceText.replace(/[^\d]/g, '');
if (cleaned) priceAmounts.push(cleaned);
});
if (priceAmounts.length === 0) return { currentPrice: null, maxPrice: null };
if (priceAmounts.length === 1) return { currentPrice: priceAmounts[0], maxPrice: null };
const prices = priceAmounts.map((p) => parseInt(p, 10)).sort((a, b) => a - b);
return { currentPrice: prices[0].toString(), maxPrice: prices[prices.length - 1].toString() };
}
async crawlAkharin(): Promise<number> {
this.logger.log('Crawling Akharin...');
const result: Partial<AkharinCar>[] = [];
try {
const { data } = await axios.get(this.akharinUrl, { headers: { 'User-Agent': 'Mozilla/5.0' } });
const $ = cheerio.load(data);
$('tr').each((_, tr) => {
const tds = $(tr).find('td');
if (tds.length < 3) return;
const carName = $(tds[0]).text().trim();
if (!carName || carName === 'نام خودرو') return;
const marketText = $(tds[1]).text().trim();
const newCarText = $(tds[2]).text().trim();
const marketPrice =
marketText.split(' ')[0] === 'بدون' ? null : marketText.split(' ')[0] || null;
const newCarPrice = newCarText.split(' ')[0] || null;
result.push({
carName,
marketPrice,
newCarPrice,
crawledAt: new Date(),
});
});
const now = new Date();
for (const item of result) {
const existing = await this.akharinModel
.findOne({ carName: item.carName })
.select('marketPrice newCarPrice priceChangedAt')
.lean();
const priceChanged =
!existing ||
existing.marketPrice !== item.marketPrice ||
existing.newCarPrice !== item.newCarPrice;
await this.akharinModel.updateOne(
{ carName: item.carName },
{
$set: {
carName: item.carName,
marketPrice: item.marketPrice ?? null,
newCarPrice: item.newCarPrice ?? null,
crawledAt: now,
updatedAt: now,
priceChangedAt: priceChanged ? now : existing?.priceChangedAt ?? null,
},
$setOnInsert: { createdAt: now },
},
{ upsert: true },
);
}
this.logger.log(`Akharin: saved ${result.length} items.`);
return result.length;
} catch (err) {
this.logger.error('Akharin crawl failed', err);
return 0;
}
}
async crawlHamrah(): Promise<number> {
this.logger.log('Crawling Hamrah...');
const result: Partial<HamrahCar>[] = [];
try {
const { data } = await axios.get(this.hamrahUrl, { headers: { 'User-Agent': 'Mozilla/5.0' } });
const $ = cheerio.load(data);
$('tbody tr').each((_, tr) => {
const tds = $(tr).find('td');
const modelDivs = $(tds[0]).find('div');
const priceDivs = $(tds[1]).find('div');
const carName = $(modelDivs[0]).text().trim();
const typeAndYear = $(modelDivs[1]).text().replace(/\s+/g, ' ').trim();
const marketPrice = $(priceDivs[0]).text().trim();
result.push({ carName, typeAndYear, marketPrice, crawledAt: new Date() });
});
const now = new Date();
for (const item of result) {
const existing = await this.hamrahModel
.findOne({ carName: item.carName, typeAndYear: item.typeAndYear })
.select('marketPrice priceChangedAt')
.lean();
const priceChanged = !existing || existing.marketPrice !== item.marketPrice;
await this.hamrahModel.updateOne(
{ carName: item.carName, typeAndYear: item.typeAndYear },
{
$set: {
carName: item.carName,
typeAndYear: item.typeAndYear ?? null,
marketPrice: item.marketPrice ?? null,
crawledAt: now,
updatedAt: now,
priceChangedAt: priceChanged ? now : existing?.priceChangedAt ?? null,
},
$setOnInsert: { createdAt: now },
},
{ upsert: true },
);
}
this.logger.log(`Hamrah: saved ${result.length} items.`);
return result.length;
} catch (err) {
this.logger.error('Hamrah crawl failed', err);
return 0;
}
}
async crawlYadakmarket(): Promise<number> {
this.logger.log('Crawling Yadakmarket...');
const result: Partial<YadakmarketPart>[] = [];
let page = 1;
let hasMorePages = true;
try {
while (hasMorePages) {
const pageUrl = page === 1
? 'https://www.yadakmarket.com/لوازم-بدنه/page/1/?loop=60&woo_ajax=1'
: `https://www.yadakmarket.com/لوازم-بدنه/page/${page}/?loop=60&woo_ajax=1`;
if (page === 1 || page % 10 === 0) {
this.logger.log(`Yadakmarket progress: page ${page}`);
}
const { data } = await axios.get(pageUrl, {
headers: { 'User-Agent': 'Mozilla/5.0', 'Accept': 'application/json' },
timeout: 15000,
});
if (!data?.items || data.status !== 'have-posts') {
hasMorePages = false;
break;
}
const $ = cheerio.load(data.items);
const products = $('div.wd-product');
if (products.length === 0) {
hasMorePages = false;
break;
}
products.each((_, product) => {
const $product = $(product);
const titleLink = $product.find('h3.wd-entities-title a');
const productName = titleLink.text().trim();
const productUrl = titleLink.attr('href') || null;
const productId = $product.attr('data-id') || null;
const priceElement = $product.find('span.price').html() || null;
const price = this.parseYadakPrice(priceElement);
if (productName) {
result.push({
productName,
productUrl,
productId,
currentPrice: price.currentPrice,
maxPrice: price.maxPrice,
crawledAt: new Date(),
});
}
});
if (data.nextPage) {
page++;
await new Promise((r) => setTimeout(r, 1200));
} else {
hasMorePages = false;
}
}
const now = new Date();
for (const item of result) {
const filter = item.productId ? { productId: item.productId } : { productName: item.productName };
const existing = await this.yadakModel
.findOne(filter)
.select('currentPrice maxPrice priceChangedAt')
.lean();
const priceChanged =
!existing ||
existing.currentPrice !== item.currentPrice ||
existing.maxPrice !== item.maxPrice;
await this.yadakModel.updateOne(
filter,
{
$set: {
productName: item.productName,
productUrl: item.productUrl ?? null,
productId: item.productId ?? null,
currentPrice: item.currentPrice ?? null,
maxPrice: item.maxPrice ?? null,
crawledAt: now,
updatedAt: now,
priceChangedAt: priceChanged ? now : existing?.priceChangedAt ?? null,
},
$setOnInsert: { createdAt: now },
},
{ upsert: true },
);
}
this.logger.log(`Yadakmarket: saved ${result.length} items.`);
return result.length;
} catch (err) {
this.logger.error('Yadakmarket crawl failed', err);
return 0;
}
}
}

27
src/main.ts Normal file
View File

@@ -0,0 +1,27 @@
import 'reflect-metadata';
import { NestFactory } from '@nestjs/core';
import { ValidationPipe } from '@nestjs/common';
import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger';
import { ConfigService } from '@nestjs/config';
import { AppModule } from './app.module';
async function bootstrap() {
const app = await NestFactory.create(AppModule, { cors: true });
app.useGlobalPipes(new ValidationPipe({ transform: true, whitelist: true }));
const config = app.get(ConfigService);
const port = config.get<number>('CAR_CRAWLER_APP_PORT', 3000);
const swaggerConfig = new DocumentBuilder()
.setTitle('Price Crawler API')
.setDescription('API for accessing crawled car prices and parts data')
.setVersion('1.0.0')
.build();
const document = SwaggerModule.createDocument(app, swaggerConfig);
SwaggerModule.setup('docs', app, document);
await app.listen(port);
}
bootstrap();

24
src/schemas/akharin-car.schema.ts Normal file
View File

@@ -0,0 +1,24 @@
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
import { HydratedDocument } from 'mongoose';
export type AkharinCarDocument = HydratedDocument<AkharinCar>;
@Schema({ timestamps: true })
export class AkharinCar {
@Prop({ required: true, index: true })
carName!: string;
@Prop({ type: String, default: null })
marketPrice!: string | null;
@Prop({ type: String, default: null })
newCarPrice!: string | null;
@Prop({ type: Date, default: null })
priceChangedAt!: Date | null;
@Prop({ default: Date.now, index: true })
crawledAt!: Date;
}
export const AkharinCarSchema = SchemaFactory.createForClass(AkharinCar);

24
src/schemas/hamrah-car.schema.ts Normal file
View File

@@ -0,0 +1,24 @@
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
import { HydratedDocument } from 'mongoose';
export type HamrahCarDocument = HydratedDocument<HamrahCar>;
@Schema({ timestamps: true })
export class HamrahCar {
@Prop({ required: true, index: true })
carName!: string;
@Prop({ type: String, default: null })
typeAndYear!: string | null;
@Prop({ type: String, default: null })
marketPrice!: string | null;
@Prop({ type: Date, default: null })
priceChangedAt!: Date | null;
@Prop({ default: Date.now, index: true })
crawledAt!: Date;
}
export const HamrahCarSchema = SchemaFactory.createForClass(HamrahCar);

30
src/schemas/yadakmarket-part.schema.ts Normal file
View File

@@ -0,0 +1,30 @@
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
import { HydratedDocument } from 'mongoose';
export type YadakmarketPartDocument = HydratedDocument<YadakmarketPart>;
@Schema({ timestamps: true })
export class YadakmarketPart {
@Prop({ required: true, index: true })
productName!: string;
@Prop({ type: String, default: null })
productUrl!: string | null;
@Prop({ type: String, default: null, index: true })
productId!: string | null;
@Prop({ type: String, default: null })
currentPrice!: string | null;
@Prop({ type: String, default: null })
maxPrice!: string | null;
@Prop({ type: Date, default: null })
priceChangedAt!: Date | null;
@Prop({ default: Date.now, index: true })
crawledAt!: Date;
}
export const YadakmarketPartSchema = SchemaFactory.createForClass(YadakmarketPart);

35
test/helper.js
View File

@@ -1,35 +0,0 @@
'use strict'
// This file contains code that we reuse
// between our tests.
const { build: buildApplication } = require('fastify-cli/helper')
const path = require('node:path')
const AppPath = path.join(__dirname, '..', 'app.js')
// Fill in this config with all the configurations
// needed for testing the application
function config () {
return {}
}
// automatically build and tear down our instance
async function build (t) {
// you can set all the options supported by the fastify CLI command
const argv = [AppPath]
// fastify-plugin ensures that all decorators
// are exposed for testing purposes, this is
// different from the production setup
const app = await buildApplication(argv, config())
// close the app after we are done
t.after(() => app.close())
return app
}
module.exports = {
config,
build
}

28
test/plugins/support.test.js
View File

@@ -1,28 +0,0 @@
'use strict'
const { test } = require('node:test')
const assert = require('node:assert')
const Fastify = require('fastify')
const Support = require('../../plugins/support')
test('support works standalone', async (t) => {
const fastify = Fastify()
fastify.register(Support)
await fastify.ready()
assert.equal(fastify.someSupport(), 'hugs')
})
// You can also use plugin with opts in fastify v2
//
// test('support works standalone', (t) => {
// t.plan(2)
// const fastify = Fastify()
// fastify.register(Support)
//
// fastify.ready((err) => {
// t.error(err)
// assert.equal(fastify.someSupport(), 'hugs')
// })
// })

28
test/routes/example.test.js
View File

@@ -1,28 +0,0 @@
'use strict'
const { test } = require('node:test')
const assert = require('node:assert')
const { build } = require('../helper')
test('example is loaded', async (t) => {
const app = await build(t)
const res = await app.inject({
url: '/example'
})
assert.equal(res.payload, 'this is an example')
})
// inject callback style:
//
// test('example is loaded', (t) => {
// t.plan(2)
// const app = await build(t)
//
// app.inject({
// url: '/example'
// }, (err, res) => {
// t.error(err)
// assert.equal(res.payload, 'this is an example')
// })
// })

28
test/routes/root.test.js
View File

@@ -1,28 +0,0 @@
'use strict'
const { test } = require('node:test')
const assert = require('node:assert')
const { build } = require('../helper')
test('default root route', async (t) => {
const app = await build(t)
const res = await app.inject({
url: '/'
})
assert.deepStrictEqual(JSON.parse(res.payload), { root: true })
})
// inject callback style:
//
// test('default root route', (t) => {
// t.plan(2)
// const app = await build(t)
//
// app.inject({
// url: '/'
// }, (err, res) => {
// t.error(err)
// assert.deepStrictEqual(JSON.parse(res.payload), { root: true })
// })
// })

14
tsconfig.build.json Normal file
View File

@@ -0,0 +1,14 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"declaration": true,
"sourceMap": false
},
"exclude": [
"node_modules",
"dist",
"test",
"**/*.spec.ts",
"src/cli/**/*.ts"
]
}

19
tsconfig.json Normal file
View File

@@ -0,0 +1,19 @@
{
"compilerOptions": {
"module": "commonjs",
"declaration": true,
"removeComments": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"allowSyntheticDefaultImports": true,
"target": "ES2021",
"sourceMap": true,
"outDir": "./dist",
"baseUrl": "./",
"incremental": true,
"skipLibCheck": true,
"strict": true
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "dist", "test", "**/*.spec.ts"]
}