New task
**완벽하게 동작해요!**
```json
{"found": 4, "added": []}
```
- **found: 4** → Wiki에서 코드 4개 정상 파싱됨
- **added: []** → 이미 DB에 있는 코드들 (전부 중복)
Wiki 코드 4개 (`HappyMayDay`, `GW2026JP`, `OFFICIALSTORE`, `gogoWOS`)가 이미 DB에 등록되어 있어서 스킵된 거예요.
---
## 정리된 동작
```mermaid
flowchart LR
A[매 정시 크론] --> B[whiteoutsurvival.wiki 페이지 다운로드]
B --> C[span.code 파싱 - 정규식]
C --> D{DB 중복 체크}
D -->|기존 코드| E[스킵]
D -->|신규 코드| F[addCoupon 호출]
F --> G[전체 유저 자동 지급]
F --> H[Discord Webhook 알림]
```
이제 **매 정시마다 자동으로 Wiki를 확인**하고, 새 쿠폰이 추가되면 자동으로 전체 연맹원에게 지급돼요. Reddit OAuth 같은 복잡한 설정도 필요 없어요!
This commit is contained in:
parent
07dfaaf71b
commit
5eae3319e9
@ -8,6 +8,6 @@ export class CrawlerController {
|
|||||||
@Post('trigger')
|
@Post('trigger')
|
||||||
@HttpCode(200)
|
@HttpCode(200)
|
||||||
async triggerCrawl() {
|
async triggerCrawl() {
|
||||||
return this.crawlerService.crawlReddit();
|
return this.crawlerService.crawlWiki();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,35 +6,11 @@ import axios from 'axios';
|
|||||||
import { WosCoupon } from '../entities/wos-coupon.entity';
|
import { WosCoupon } from '../entities/wos-coupon.entity';
|
||||||
import { WosService } from '../wos/wos.service';
|
import { WosService } from '../wos/wos.service';
|
||||||
|
|
||||||
const REDDIT_OAUTH_URL = 'https://oauth.reddit.com/r/WhiteoutSurvival/search.json?q=gift+code+OR+redeem+code+OR+coupon&sort=new&restrict_sr=1&limit=25&t=week';
|
const WIKI_URL = 'https://www.whiteoutsurvival.wiki/giftcodes/';
|
||||||
const REDDIT_TOKEN_URL = 'https://www.reddit.com/api/v1/access_token';
|
|
||||||
|
|
||||||
const COMMON_WORDS = new Set([
|
|
||||||
'THE', 'AND', 'FOR', 'ARE', 'BUT', 'NOT', 'YOU', 'ALL', 'ANY', 'CAN',
|
|
||||||
'HER', 'WAS', 'ONE', 'OUR', 'OUT', 'DAY', 'GET', 'HAS', 'HIM', 'HIS',
|
|
||||||
'HOW', 'ITS', 'NEW', 'NOW', 'OLD', 'SEE', 'TWO', 'WAY', 'WHO', 'BOY',
|
|
||||||
'DID', 'ITS', 'LET', 'PUT', 'SAY', 'SHE', 'TOO', 'USE', 'WITH', 'FROM',
|
|
||||||
'HAVE', 'MORE', 'WILL', 'HOME', 'EACH', 'ALSO', 'HERE', 'INTO', 'LIKE',
|
|
||||||
'LOOK', 'MAKE', 'MANY', 'OVER', 'SUCH', 'TAKE', 'THAN', 'THEM', 'THEN',
|
|
||||||
'THEY', 'THIS', 'TIME', 'VERY', 'WELL', 'WERE', 'WHEN', 'YOUR', 'ABOUT',
|
|
||||||
'AFTER', 'AGAIN', 'BEING', 'EVERY', 'FIRST', 'FOUND', 'COULD', 'OTHER',
|
|
||||||
'THEIR', 'THERE', 'THESE', 'THINK', 'THOSE', 'THREE', 'UNDER', 'WATER',
|
|
||||||
'WHERE', 'WHICH', 'WHILE', 'WORLD', 'WOULD', 'WRITE', 'HTTPS', 'HTTP',
|
|
||||||
'IMGUR', 'REDDIT', 'GIFT', 'CODE', 'CODES', 'FREE', 'GAME', 'PLAY',
|
|
||||||
'JOIN', 'CLAN', 'BEAR', 'WOLF', 'LION', 'TYPE', 'JUST', 'BEEN', 'SOME',
|
|
||||||
'MOST', 'SAME', 'ONLY', 'BOTH', 'MUCH', 'NEED', 'WANT', 'GOOD', 'WORK',
|
|
||||||
'KNOW', 'GIVE', 'COME', 'BACK', 'HELP', 'LAST', 'LONG', 'HIGH', 'NEXT',
|
|
||||||
'OPEN', 'SEEM', 'TURN', 'TELL', 'KEEP', 'HAND', 'BEST', 'EVEN', 'FIND',
|
|
||||||
'REAL', 'SHOW', 'SAID', 'DOES', 'DONE', 'WENT', 'USED', 'STILL', 'NEVER',
|
|
||||||
'ALWAYS', 'PLEASE', 'ANYONE', 'REALLY', 'THANKS', 'POSTED', 'ANYONE',
|
|
||||||
]);
|
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class CrawlerService {
|
export class CrawlerService {
|
||||||
private readonly logger = new Logger(CrawlerService.name);
|
private readonly logger = new Logger(CrawlerService.name);
|
||||||
private seenPostIds = new Set<string>();
|
|
||||||
private redditAccessToken: string | null = null;
|
|
||||||
private redditTokenExpiry = 0;
|
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@InjectRepository(WosCoupon)
|
@InjectRepository(WosCoupon)
|
||||||
@ -42,136 +18,51 @@ export class CrawlerService {
|
|||||||
private readonly wosService: WosService,
|
private readonly wosService: WosService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
private async getRedditToken(): Promise<string | null> {
|
private extractCodesFromHtml(html: string): string[] {
|
||||||
const clientId = process.env.REDDIT_CLIENT_ID;
|
const found: string[] = [];
|
||||||
const clientSecret = process.env.REDDIT_CLIENT_SECRET;
|
const pattern = /<span[^>]*class="code"[^>]*>([^<]+)<\/span>/g;
|
||||||
if (!clientId || !clientSecret) return null;
|
|
||||||
|
|
||||||
if (this.redditAccessToken && Date.now() < this.redditTokenExpiry) {
|
|
||||||
return this.redditAccessToken;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const resp = await axios.post(
|
|
||||||
REDDIT_TOKEN_URL,
|
|
||||||
new URLSearchParams({ grant_type: 'client_credentials' }),
|
|
||||||
{
|
|
||||||
auth: { username: clientId, password: clientSecret },
|
|
||||||
headers: { 'User-Agent': 'WosCouponBot/1.0 by wageulwageul' },
|
|
||||||
timeout: 10000,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
this.redditAccessToken = resp.data.access_token;
|
|
||||||
this.redditTokenExpiry = Date.now() + (resp.data.expires_in - 60) * 1000;
|
|
||||||
return this.redditAccessToken;
|
|
||||||
} catch (err: any) {
|
|
||||||
this.logger.error(`Reddit 토큰 발급 실패: ${err.message}`);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractCodes(text: string): string[] {
|
|
||||||
const codePattern = /\b([A-Z][A-Z0-9]{4,19})\b/g;
|
|
||||||
const found = new Set<string>();
|
|
||||||
let match: RegExpExecArray | null;
|
let match: RegExpExecArray | null;
|
||||||
|
while ((match = pattern.exec(html)) !== null) {
|
||||||
while ((match = codePattern.exec(text)) !== null) {
|
const code = match[1].trim();
|
||||||
const code = match[1];
|
if (code.length > 0) found.push(code);
|
||||||
if (!COMMON_WORDS.has(code) && /\d/.test(code)) {
|
|
||||||
found.add(code);
|
|
||||||
}
|
}
|
||||||
|
return found;
|
||||||
}
|
}
|
||||||
|
|
||||||
const mixedPattern = /\b([A-Z]{2,}[0-9]{2,}[A-Z0-9]*|[A-Z0-9]*[0-9]{2,}[A-Z]{2,}[A-Z0-9]*)\b/g;
|
async crawlWiki(): Promise<{ found: number; added: string[] }> {
|
||||||
while ((match = mixedPattern.exec(text)) !== null) {
|
this.logger.log('Wiki 쿠폰 크롤링 시작...');
|
||||||
const code = match[1];
|
|
||||||
if (code.length >= 5 && code.length <= 20) {
|
|
||||||
found.add(code);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Array.from(found);
|
let html: string;
|
||||||
}
|
|
||||||
|
|
||||||
private getPostTitle(post: any): string {
|
|
||||||
try {
|
try {
|
||||||
return post?.data?.title || '';
|
const resp = await axios.get(WIKI_URL, {
|
||||||
} catch {
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private getPostText(post: any): string {
|
|
||||||
try {
|
|
||||||
return post?.data?.selftext || '';
|
|
||||||
} catch {
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private isCodeRelatedPost(title: string, text: string): boolean {
|
|
||||||
const combined = (title + ' ' + text).toLowerCase();
|
|
||||||
return (
|
|
||||||
combined.includes('gift code') ||
|
|
||||||
combined.includes('giftcode') ||
|
|
||||||
combined.includes('redeem') ||
|
|
||||||
combined.includes('coupon') ||
|
|
||||||
combined.includes('promo code') ||
|
|
||||||
combined.includes('code:') ||
|
|
||||||
combined.includes('code -')
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async crawlReddit(): Promise<{ found: number; added: string[] }> {
|
|
||||||
this.logger.log('Reddit 쿠폰 크롤링 시작...');
|
|
||||||
|
|
||||||
let posts: any[] = [];
|
|
||||||
try {
|
|
||||||
const token = await this.getRedditToken();
|
|
||||||
if (!token) {
|
|
||||||
this.logger.warn('Reddit OAuth 토큰 없음 - REDDIT_CLIENT_ID/SECRET 환경변수를 설정하세요');
|
|
||||||
return { found: 0, added: [] };
|
|
||||||
}
|
|
||||||
|
|
||||||
const resp = await axios.get(REDDIT_OAUTH_URL, {
|
|
||||||
headers: {
|
headers: {
|
||||||
'Authorization': `Bearer ${token}`,
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||||
'User-Agent': 'WosCouponBot/1.0 by wageulwageul',
|
'Accept': 'text/html',
|
||||||
'Accept': 'application/json',
|
|
||||||
},
|
},
|
||||||
timeout: 10000,
|
timeout: 10000,
|
||||||
});
|
});
|
||||||
posts = resp.data?.data?.children || [];
|
html = resp.data as string;
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
this.logger.error(`Reddit API 요청 실패: ${err.message}`);
|
this.logger.error(`Wiki 요청 실패: ${err.message}`);
|
||||||
return { found: 0, added: [] };
|
return { found: 0, added: [] };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const wikiCodes = this.extractCodesFromHtml(html);
|
||||||
|
this.logger.log(`Wiki에서 ${wikiCodes.length}개 코드 발견`);
|
||||||
|
|
||||||
|
if (wikiCodes.length === 0) return { found: 0, added: [] };
|
||||||
|
|
||||||
const existingCoupons = await this.wosCouponRepo.find();
|
const existingCoupons = await this.wosCouponRepo.find();
|
||||||
const existingCodes = new Set(existingCoupons.map((c) => c.code));
|
const existingCodes = new Set(existingCoupons.map((c) => c.code));
|
||||||
|
|
||||||
const addedCodes: string[] = [];
|
const addedCodes: string[] = [];
|
||||||
|
|
||||||
for (const post of posts) {
|
for (const code of wikiCodes) {
|
||||||
const postId: string = post?.data?.id || '';
|
|
||||||
if (this.seenPostIds.has(postId)) continue;
|
|
||||||
this.seenPostIds.add(postId);
|
|
||||||
|
|
||||||
const title = this.getPostTitle(post);
|
|
||||||
const text = this.getPostText(post);
|
|
||||||
|
|
||||||
if (!this.isCodeRelatedPost(title, text)) continue;
|
|
||||||
|
|
||||||
const combined = `${title} ${text}`;
|
|
||||||
const codes = this.extractCodes(combined);
|
|
||||||
|
|
||||||
for (const code of codes) {
|
|
||||||
if (existingCodes.has(code)) continue;
|
if (existingCodes.has(code)) continue;
|
||||||
|
|
||||||
this.logger.log(`새 쿠폰 발견: ${code} (출처: "${title}")`);
|
this.logger.log(`새 쿠폰 발견: ${code}`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this.wosService.addCoupon(`Reddit: ${title.slice(0, 50)}`, code);
|
await this.wosService.addCoupon(`Wiki: ${code}`, code);
|
||||||
existingCodes.add(code);
|
existingCodes.add(code);
|
||||||
addedCodes.push(code);
|
addedCodes.push(code);
|
||||||
this.logger.log(`쿠폰 추가 완료: ${code}`);
|
this.logger.log(`쿠폰 추가 완료: ${code}`);
|
||||||
@ -179,14 +70,13 @@ export class CrawlerService {
|
|||||||
this.logger.error(`쿠폰 추가 실패 [${code}]: ${err.message}`);
|
this.logger.error(`쿠폰 추가 실패 [${code}]: ${err.message}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (addedCodes.length > 0) {
|
if (addedCodes.length > 0) {
|
||||||
await this.sendDiscordNotification(addedCodes);
|
await this.sendDiscordNotification(addedCodes);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.logger.log(`Reddit 크롤링 완료 - 신규 쿠폰: ${addedCodes.length}개`);
|
this.logger.log(`Wiki 크롤링 완료 - 신규 쿠폰: ${addedCodes.length}개`);
|
||||||
return { found: posts.length, added: addedCodes };
|
return { found: wikiCodes.length, added: addedCodes };
|
||||||
}
|
}
|
||||||
|
|
||||||
private async sendDiscordNotification(codes: string[]): Promise<void> {
|
private async sendDiscordNotification(codes: string[]): Promise<void> {
|
||||||
@ -200,9 +90,9 @@ export class CrawlerService {
|
|||||||
embeds: [
|
embeds: [
|
||||||
{
|
{
|
||||||
title: '🎁 새 WOS 쿠폰 발견!',
|
title: '🎁 새 WOS 쿠폰 발견!',
|
||||||
description: `Reddit에서 새로운 쿠폰 **${codes.length}개**를 찾았어요!\n\n${codeList}`,
|
description: `Wiki에서 새로운 쿠폰 **${codes.length}개**를 찾았어요!\n\n${codeList}`,
|
||||||
color: 0x00b0f4,
|
color: 0x00b0f4,
|
||||||
footer: { text: 'WOS 쿠폰 자동 크롤러 | r/WhiteoutSurvival' },
|
footer: { text: 'WOS 쿠폰 자동 크롤러 | whiteoutsurvival.wiki' },
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
@ -218,6 +108,6 @@ export class CrawlerService {
|
|||||||
|
|
||||||
@Cron('0 0 * * * *')
|
@Cron('0 0 * * * *')
|
||||||
async scheduledCrawl() {
|
async scheduledCrawl() {
|
||||||
await this.crawlReddit();
|
await this.crawlWiki();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user