diff --git a/back/src/crawler/crawler.controller.ts b/back/src/crawler/crawler.controller.ts index 5cf3056..425c1d5 100644 --- a/back/src/crawler/crawler.controller.ts +++ b/back/src/crawler/crawler.controller.ts @@ -8,6 +8,6 @@ export class CrawlerController { @Post('trigger') @HttpCode(200) async triggerCrawl() { - return this.crawlerService.crawlReddit(); + return this.crawlerService.crawlWiki(); } } diff --git a/back/src/crawler/crawler.service.ts b/back/src/crawler/crawler.service.ts index c5c9148..248b6a3 100644 --- a/back/src/crawler/crawler.service.ts +++ b/back/src/crawler/crawler.service.ts @@ -6,35 +6,11 @@ import axios from 'axios'; import { WosCoupon } from '../entities/wos-coupon.entity'; import { WosService } from '../wos/wos.service'; -const REDDIT_OAUTH_URL = 'https://oauth.reddit.com/r/WhiteoutSurvival/search.json?q=gift+code+OR+redeem+code+OR+coupon&sort=new&restrict_sr=1&limit=25&t=week'; -const REDDIT_TOKEN_URL = 'https://www.reddit.com/api/v1/access_token'; - -const COMMON_WORDS = new Set([ - 'THE', 'AND', 'FOR', 'ARE', 'BUT', 'NOT', 'YOU', 'ALL', 'ANY', 'CAN', - 'HER', 'WAS', 'ONE', 'OUR', 'OUT', 'DAY', 'GET', 'HAS', 'HIM', 'HIS', - 'HOW', 'ITS', 'NEW', 'NOW', 'OLD', 'SEE', 'TWO', 'WAY', 'WHO', 'BOY', - 'DID', 'ITS', 'LET', 'PUT', 'SAY', 'SHE', 'TOO', 'USE', 'WITH', 'FROM', - 'HAVE', 'MORE', 'WILL', 'HOME', 'EACH', 'ALSO', 'HERE', 'INTO', 'LIKE', - 'LOOK', 'MAKE', 'MANY', 'OVER', 'SUCH', 'TAKE', 'THAN', 'THEM', 'THEN', - 'THEY', 'THIS', 'TIME', 'VERY', 'WELL', 'WERE', 'WHEN', 'YOUR', 'ABOUT', - 'AFTER', 'AGAIN', 'BEING', 'EVERY', 'FIRST', 'FOUND', 'COULD', 'OTHER', - 'THEIR', 'THERE', 'THESE', 'THINK', 'THOSE', 'THREE', 'UNDER', 'WATER', - 'WHERE', 'WHICH', 'WHILE', 'WORLD', 'WOULD', 'WRITE', 'HTTPS', 'HTTP', - 'IMGUR', 'REDDIT', 'GIFT', 'CODE', 'CODES', 'FREE', 'GAME', 'PLAY', - 'JOIN', 'CLAN', 'BEAR', 'WOLF', 'LION', 'TYPE', 'JUST', 'BEEN', 'SOME', - 'MOST', 'SAME', 'ONLY', 'BOTH', 'MUCH', 'NEED', 'WANT', 'GOOD', 'WORK', - 'KNOW', 'GIVE', 'COME', 'BACK', 'HELP', 'LAST', 'LONG', 'HIGH', 'NEXT', - 'OPEN', 'SEEM', 'TURN', 'TELL', 'KEEP', 'HAND', 'BEST', 'EVEN', 'FIND', - 'REAL', 'SHOW', 'SAID', 'DOES', 'DONE', 'WENT', 'USED', 'STILL', 'NEVER', - 'ALWAYS', 'PLEASE', 'ANYONE', 'REALLY', 'THANKS', 'POSTED', 'ANYONE', -]); +const WIKI_URL = 'https://www.whiteoutsurvival.wiki/giftcodes/'; @Injectable() export class CrawlerService { private readonly logger = new Logger(CrawlerService.name); - private seenPostIds = new Set(); - private redditAccessToken: string | null = null; - private redditTokenExpiry = 0; constructor( @InjectRepository(WosCoupon) @@ -42,142 +18,56 @@ export class CrawlerService { private readonly wosService: WosService, ) {} - private async getRedditToken(): Promise { - const clientId = process.env.REDDIT_CLIENT_ID; - const clientSecret = process.env.REDDIT_CLIENT_SECRET; - if (!clientId || !clientSecret) return null; - - if (this.redditAccessToken && Date.now() < this.redditTokenExpiry) { - return this.redditAccessToken; - } - - try { - const resp = await axios.post( - REDDIT_TOKEN_URL, - new URLSearchParams({ grant_type: 'client_credentials' }), - { - auth: { username: clientId, password: clientSecret }, - headers: { 'User-Agent': 'WosCouponBot/1.0 by wageulwageul' }, - timeout: 10000, - }, - ); - this.redditAccessToken = resp.data.access_token; - this.redditTokenExpiry = Date.now() + (resp.data.expires_in - 60) * 1000; - return this.redditAccessToken; - } catch (err: any) { - this.logger.error(`Reddit 토큰 발급 실패: ${err.message}`); - return null; - } - } - - private extractCodes(text: string): string[] { - const codePattern = /\b([A-Z][A-Z0-9]{4,19})\b/g; - const found = new Set(); + private extractCodesFromHtml(html: string): string[] { + const found: string[] = []; + const pattern = /]*class="code"[^>]*>([^<]+)<\/span>/g; let match: RegExpExecArray | null; - - while ((match = codePattern.exec(text)) !== null) { - const code = match[1]; - if (!COMMON_WORDS.has(code) && /\d/.test(code)) { - found.add(code); - } + while ((match = pattern.exec(html)) !== null) { + const code = match[1].trim(); + if (code.length > 0) found.push(code); } - - const mixedPattern = /\b([A-Z]{2,}[0-9]{2,}[A-Z0-9]*|[A-Z0-9]*[0-9]{2,}[A-Z]{2,}[A-Z0-9]*)\b/g; - while ((match = mixedPattern.exec(text)) !== null) { - const code = match[1]; - if (code.length >= 5 && code.length <= 20) { - found.add(code); - } - } - - return Array.from(found); + return found; } - private getPostTitle(post: any): string { + async crawlWiki(): Promise<{ found: number; added: string[] }> { + this.logger.log('Wiki 쿠폰 크롤링 시작...'); + + let html: string; try { - return post?.data?.title || ''; - } catch { - return ''; - } - } - - private getPostText(post: any): string { - try { - return post?.data?.selftext || ''; - } catch { - return ''; - } - } - - private isCodeRelatedPost(title: string, text: string): boolean { - const combined = (title + ' ' + text).toLowerCase(); - return ( - combined.includes('gift code') || - combined.includes('giftcode') || - combined.includes('redeem') || - combined.includes('coupon') || - combined.includes('promo code') || - combined.includes('code:') || - combined.includes('code -') - ); - } - - async crawlReddit(): Promise<{ found: number; added: string[] }> { - this.logger.log('Reddit 쿠폰 크롤링 시작...'); - - let posts: any[] = []; - try { - const token = await this.getRedditToken(); - if (!token) { - this.logger.warn('Reddit OAuth 토큰 없음 - REDDIT_CLIENT_ID/SECRET 환경변수를 설정하세요'); - return { found: 0, added: [] }; - } - - const resp = await axios.get(REDDIT_OAUTH_URL, { + const resp = await axios.get(WIKI_URL, { headers: { - 'Authorization': `Bearer ${token}`, - 'User-Agent': 'WosCouponBot/1.0 by wageulwageul', - 'Accept': 'application/json', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', + 'Accept': 'text/html', }, timeout: 10000, }); - posts = resp.data?.data?.children || []; + html = resp.data as string; } catch (err: any) { - this.logger.error(`Reddit API 요청 실패: ${err.message}`); + this.logger.error(`Wiki 요청 실패: ${err.message}`); return { found: 0, added: [] }; } + const wikiCodes = this.extractCodesFromHtml(html); + this.logger.log(`Wiki에서 ${wikiCodes.length}개 코드 발견`); + + if (wikiCodes.length === 0) return { found: 0, added: [] }; + const existingCoupons = await this.wosCouponRepo.find(); const existingCodes = new Set(existingCoupons.map((c) => c.code)); const addedCodes: string[] = []; - for (const post of posts) { - const postId: string = post?.data?.id || ''; - if (this.seenPostIds.has(postId)) continue; - this.seenPostIds.add(postId); + for (const code of wikiCodes) { + if (existingCodes.has(code)) continue; - const title = this.getPostTitle(post); - const text = this.getPostText(post); - - if (!this.isCodeRelatedPost(title, text)) continue; - - const combined = `${title} ${text}`; - const codes = this.extractCodes(combined); - - for (const code of codes) { - if (existingCodes.has(code)) continue; - - this.logger.log(`새 쿠폰 발견: ${code} (출처: "${title}")`); - - try { - await this.wosService.addCoupon(`Reddit: ${title.slice(0, 50)}`, code); - existingCodes.add(code); - addedCodes.push(code); - this.logger.log(`쿠폰 추가 완료: ${code}`); - } catch (err: any) { - this.logger.error(`쿠폰 추가 실패 [${code}]: ${err.message}`); - } + this.logger.log(`새 쿠폰 발견: ${code}`); + try { + await this.wosService.addCoupon(`Wiki: ${code}`, code); + existingCodes.add(code); + addedCodes.push(code); + this.logger.log(`쿠폰 추가 완료: ${code}`); + } catch (err: any) { + this.logger.error(`쿠폰 추가 실패 [${code}]: ${err.message}`); } } @@ -185,8 +75,8 @@ export class CrawlerService { await this.sendDiscordNotification(addedCodes); } - this.logger.log(`Reddit 크롤링 완료 - 신규 쿠폰: ${addedCodes.length}개`); - return { found: posts.length, added: addedCodes }; + this.logger.log(`Wiki 크롤링 완료 - 신규 쿠폰: ${addedCodes.length}개`); + return { found: wikiCodes.length, added: addedCodes }; } private async sendDiscordNotification(codes: string[]): Promise { @@ -200,9 +90,9 @@ export class CrawlerService { embeds: [ { title: '🎁 새 WOS 쿠폰 발견!', - description: `Reddit에서 새로운 쿠폰 **${codes.length}개**를 찾았어요!\n\n${codeList}`, + description: `Wiki에서 새로운 쿠폰 **${codes.length}개**를 찾았어요!\n\n${codeList}`, color: 0x00b0f4, - footer: { text: 'WOS 쿠폰 자동 크롤러 | r/WhiteoutSurvival' }, + footer: { text: 'WOS 쿠폰 자동 크롤러 | whiteoutsurvival.wiki' }, timestamp: new Date().toISOString(), }, ], @@ -218,6 +108,6 @@ export class CrawlerService { @Cron('0 0 * * * *') async scheduledCrawl() { - await this.crawlReddit(); + await this.crawlWiki(); } }