commit
7d9e806d75
35 changed files with 1897 additions and 0 deletions
@ -0,0 +1,131 @@ |
|||||
|
### JetBrains template |
||||
|
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm |
||||
|
|
||||
|
*.iml |
||||
|
|
||||
|
## Directory-based project format: |
||||
|
.idea/ |
||||
|
/out/ |
||||
|
|
||||
|
|
||||
|
### NetBeans template |
||||
|
nbproject/private/ |
||||
|
build/ |
||||
|
nbbuild/ |
||||
|
dist/ |
||||
|
nbdist/ |
||||
|
nbactions.xml |
||||
|
nb-configuration.xml |
||||
|
.nb-gradle/ |
||||
|
|
||||
|
|
||||
|
### Eclipse template |
||||
|
*.pydevproject |
||||
|
.metadata |
||||
|
.gradle |
||||
|
bin/ |
||||
|
tmp/ |
||||
|
*.tmp |
||||
|
*.bak |
||||
|
*.swp |
||||
|
*~.nib |
||||
|
local.properties |
||||
|
.settings/ |
||||
|
.loadpath |
||||
|
|
||||
|
# Eclipse Core |
||||
|
.project |
||||
|
|
||||
|
# External tool builders |
||||
|
.externalToolBuilders/ |
||||
|
|
||||
|
# Locally stored "Eclipse launch configurations" |
||||
|
*.launch |
||||
|
|
||||
|
# CDT-specific |
||||
|
.cproject |
||||
|
|
||||
|
# JDT-specific (Eclipse Java Development Tools) |
||||
|
.classpath |
||||
|
|
||||
|
# PDT-specific |
||||
|
.buildpath |
||||
|
|
||||
|
# sbteclipse plugin |
||||
|
.target |
||||
|
|
||||
|
# TeXlipse plugin |
||||
|
.texlipse |
||||
|
|
||||
|
|
||||
|
### Maven template |
||||
|
target/ |
||||
|
pom.xml.tag |
||||
|
pom.xml.releaseBackup |
||||
|
pom.xml.versionsBackup |
||||
|
pom.xml.next |
||||
|
release.properties |
||||
|
dependency-reduced-pom.xml |
||||
|
buildNumber.properties |
||||
|
.mvn/timing.properties |
||||
|
|
||||
|
|
||||
|
### Linux template |
||||
|
*~ |
||||
|
|
||||
|
# KDE directory preferences |
||||
|
.directory |
||||
|
|
||||
|
# Linux trash folder which might appear on any partition or disk |
||||
|
.Trash-* |
||||
|
|
||||
|
|
||||
|
### Windows template |
||||
|
# Windows image file caches |
||||
|
Thumbs.db |
||||
|
ehthumbs.db |
||||
|
|
||||
|
# Folder config file |
||||
|
Desktop.ini |
||||
|
|
||||
|
# Recycle Bin used on file shares |
||||
|
$RECYCLE.BIN/ |
||||
|
|
||||
|
# Windows Installer files |
||||
|
*.cab |
||||
|
*.msi |
||||
|
*.msm |
||||
|
*.msp |
||||
|
|
||||
|
# Windows shortcuts |
||||
|
*.lnk |
||||
|
|
||||
|
|
||||
|
### OSX template |
||||
|
.DS_Store |
||||
|
.AppleDouble |
||||
|
.LSOverride |
||||
|
|
||||
|
# Icon must end with two \r |
||||
|
Icon |
||||
|
|
||||
|
# Thumbnails |
||||
|
._* |
||||
|
|
||||
|
# Files that might appear in the root of a volume |
||||
|
.DocumentRevisions-V100 |
||||
|
.fseventsd |
||||
|
.Spotlight-V100 |
||||
|
.TemporaryItems |
||||
|
.Trashes |
||||
|
.VolumeIcon.icns |
||||
|
|
||||
|
# Directories potentially created on remote AFP share |
||||
|
.AppleDB |
||||
|
.AppleDesktop |
||||
|
Network Trash Folder |
||||
|
Temporary Items |
||||
|
.apdisk |
||||
|
|
||||
|
/secrets |
||||
|
/dump* |
||||
@ -0,0 +1,10 @@ |
|||||
|
FROM openjdk:8-jre-alpine as build |
||||
|
WORKDIR /workspace/app |
||||
|
|
||||
|
WORKDIR /topic-starters-app |
||||
|
COPY ./api/target/topicstarters-api.jar app.jar |
||||
|
COPY ./run.sh run.sh |
||||
|
|
||||
|
RUN chmod +x run.sh |
||||
|
|
||||
|
ENTRYPOINT ["./run.sh"] |
||||
@ -0,0 +1,10 @@ |
|||||
|
build: |
||||
|
@docker-compose -p topicstarters build; |
||||
|
run: |
||||
|
@docker-compose -p topicstarters up -d |
||||
|
stop: |
||||
|
@docker-compose -p topicstarters down |
||||
|
clean-data: |
||||
|
@docker-compose -p topicstarters down -v |
||||
|
clean-images: |
||||
|
@docker rmi `docker images -q -f "dangling=true"` |
||||
@ -0,0 +1,107 @@ |
|||||
|
# Thmmy Topic Starters |
||||
|
> A service that parses all the topics of thmmy.gr into a database and exposes an endpoint for getting filtered pages of them |
||||
|
|
||||
|
Thmmy topic starters is an application that crawls all thmmy.gr boards every day at 2 a.m. Information parsed are then saved in a |
||||
|
postgres database and can be accessed by the endpoint `/api/topicstarters`. |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
# API endpoint |
||||
|
|
||||
|
## View topic starters |
||||
|
|
||||
|
``` |
||||
|
GET /api/topicstarters |
||||
|
``` |
||||
|
|
||||
|
### Parameters |
||||
|
|
||||
|
| Name | Type | Description | |
||||
|
| ----- | ------ | ----------- | |
||||
|
| user | String | **Optional**. The username or ID of the user. Filters the results by user. | |
||||
|
| board | String | **Optional**. The title or ID of the board. Filters the results by board. | |
||||
|
| topic | String | **Optional**. The subject or ID of the topic. Filters the results by topic. | |
||||
|
|
||||
|
#### Example |
||||
|
|
||||
|
```shell script |
||||
|
curl --location \ |
||||
|
--request GET 'localhost:8080/api/topicstarters' \ |
||||
|
--form 'user=14670' \ |
||||
|
--form 'board=Ανακοινώσεις και Έκτακτα νέα' \ |
||||
|
--form 'topic=68000' |
||||
|
``` |
||||
|
|
||||
|
### Response |
||||
|
|
||||
|
``` |
||||
|
Status: 200 OK |
||||
|
Content-Type: application/json;charset=UTF-8 |
||||
|
Content-Length: 962 |
||||
|
Content-Encoding: gzip |
||||
|
``` |
||||
|
```json |
||||
|
{ |
||||
|
"content": [ |
||||
|
{ |
||||
|
"id": "d806599f-ae77-4780-bd3d-510943588054", |
||||
|
"topicId": 68000, |
||||
|
"topicUrl": "https://www.thmmy.gr/smf/index.php?topic=68000.0", |
||||
|
"starterUsername": "Apostolof", |
||||
|
"starterUrl": "https://www.thmmy.gr/smf/index.php?action=profile;u=14670", |
||||
|
"starterId": 14670, |
||||
|
"boardTitle": "Ανακοινώσεις και Έκτακτα νέα", |
||||
|
"boardUrl": "https://www.thmmy.gr/smf/index.php?board=25.0", |
||||
|
"boardId": 25, |
||||
|
"topicSubject": "mTHMMY (alpha version)", |
||||
|
"numberOfReplies": 175, |
||||
|
"numberOfViews": 15729 |
||||
|
} |
||||
|
], |
||||
|
"pageable": { |
||||
|
[...] |
||||
|
}, |
||||
|
[...] |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
\* part of the response truncated for brevity |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
# Build docker image |
||||
|
|
||||
|
To build the docker image you first need to build the java application for production: |
||||
|
```shell script |
||||
|
mvn clean install package |
||||
|
``` |
||||
|
|
||||
|
Define a username, password and database name for the postgres database in the file `./env/topic_starters_postgres.env`. |
||||
|
An example of what this file might look like is given in `./env/topic_starters_postgres.example.env`. |
||||
|
|
||||
|
If you want to get all the topics accessible by a logged-in user (rather that just those publicly available to guests) you also need to create two more files containing the username and password of a user for the application to use. |
||||
|
* `./secrets/username`: which should contain the username |
||||
|
* `./secrets/password`: which should contain the password |
||||
|
|
||||
|
Then just use the Makefile to handle the build: |
||||
|
```shell script |
||||
|
make build |
||||
|
``` |
||||
|
|
||||
|
Run the image using: |
||||
|
```shell script |
||||
|
make run |
||||
|
``` |
||||
|
|
||||
|
Stop the container using: |
||||
|
```shell script |
||||
|
make stop |
||||
|
``` |
||||
|
|
||||
|
The Makefile also provides targets for cleaning the data and dangling images. |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## License |
||||
|
|
||||
|
[](https://gitlab.com/Apostolof/flavours-without-borders/blob/master/LICENSE.md) |
||||
@ -0,0 +1,103 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<parent> |
||||
|
<artifactId>topicstarter</artifactId> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<version>1.0.0</version> |
||||
|
</parent> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
|
||||
|
<artifactId>api</artifactId> |
||||
|
|
||||
|
<properties> |
||||
|
<start-class>gr.thmmy.mthmmy.topicstarter.api.TopicStarterApplication</start-class> |
||||
|
</properties> |
||||
|
|
||||
|
<dependencies> |
||||
|
<!-- Module Dependencies --> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>entity</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>repository</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>scheduled</artifactId> |
||||
|
<scope>compile</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>service</artifactId> |
||||
|
</dependency> |
||||
|
|
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-configuration-processor</artifactId> |
||||
|
<optional>true</optional> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Web Dependencies --> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter-web</artifactId> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Utility Dependencies--> |
||||
|
<dependency> |
||||
|
<groupId>org.projectlombok</groupId> |
||||
|
<artifactId>lombok</artifactId> |
||||
|
<scope>provided</scope> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Database Dependencies --> |
||||
|
<dependency> |
||||
|
<groupId>org.postgresql</groupId> |
||||
|
<artifactId>postgresql</artifactId> |
||||
|
<scope>runtime</scope> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
|
||||
|
<build> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-maven-plugin</artifactId> |
||||
|
<configuration> |
||||
|
<finalName>topicstarters</finalName> |
||||
|
<classifier>api</classifier> |
||||
|
<executable>true</executable> |
||||
|
</configuration> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-source-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-sources</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-javadoc-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-javadocs</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
</project> |
||||
@ -0,0 +1,18 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.api; |
||||
|
|
||||
|
import gr.thmmy.mthmmy.topicstarter.TopicStarterRepositoryConfig; |
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.TopicStarterEntityConfiguration; |
||||
|
import gr.thmmy.mthmmy.topicstarter.scheduled.TopicStarterSchedulesConfig; |
||||
|
import gr.thmmy.mthmmy.topicstarter.service.TopicStarterServiceConfiguration; |
||||
|
import org.springframework.context.annotation.Configuration; |
||||
|
import org.springframework.context.annotation.Import; |
||||
|
|
||||
|
@Configuration |
||||
|
@Import({ |
||||
|
TopicStarterServiceConfiguration.class, |
||||
|
TopicStarterRepositoryConfig.class, |
||||
|
TopicStarterEntityConfiguration.class, |
||||
|
TopicStarterSchedulesConfig.class |
||||
|
}) |
||||
|
public class TopicStarterApiConfig { |
||||
|
} |
||||
@ -0,0 +1,19 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.api; |
||||
|
|
||||
|
import org.springframework.boot.SpringApplication; |
||||
|
import org.springframework.boot.autoconfigure.SpringBootApplication; |
||||
|
import org.springframework.boot.builder.SpringApplicationBuilder; |
||||
|
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; |
||||
|
|
||||
|
@SpringBootApplication |
||||
|
public class TopicStarterApplication extends SpringBootServletInitializer { |
||||
|
|
||||
|
public static void main(final String... args) { |
||||
|
SpringApplication.run(TopicStarterApplication.class, args); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected SpringApplicationBuilder configure(final SpringApplicationBuilder application) { |
||||
|
return application.sources(TopicStarterApplication.class); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,14 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.api.controller.topic.starter; |
||||
|
|
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
||||
|
import org.springframework.data.domain.Page; |
||||
|
import org.springframework.data.domain.Pageable; |
||||
|
import org.springframework.http.ResponseEntity; |
||||
|
|
||||
|
public interface TopicStarterController { |
||||
|
|
||||
|
ResponseEntity<Page<TopicStarter>> topics(String user, |
||||
|
String board, |
||||
|
String topic, |
||||
|
Pageable pageable); |
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.api.controller.topic.starter; |
||||
|
|
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
||||
|
import gr.thmmy.mthmmy.topicstarter.service.topic.starter.TopicStarterService; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.data.domain.Page; |
||||
|
import org.springframework.data.domain.Pageable; |
||||
|
import org.springframework.http.ResponseEntity; |
||||
|
import org.springframework.web.bind.annotation.GetMapping; |
||||
|
import org.springframework.web.bind.annotation.RequestMapping; |
||||
|
import org.springframework.web.bind.annotation.RequestParam; |
||||
|
import org.springframework.web.bind.annotation.RestController; |
||||
|
|
||||
|
@RestController |
||||
|
@RequestMapping("/api") |
||||
|
@RequiredArgsConstructor |
||||
|
@Slf4j |
||||
|
public class TopicStarterControllerImpl implements TopicStarterController { |
||||
|
|
||||
|
private final TopicStarterService topicStarterService; |
||||
|
|
||||
|
@Override |
||||
|
@GetMapping("/topicstarters") |
||||
|
public ResponseEntity<Page<TopicStarter>> topics(@RequestParam(required = false) String user, |
||||
|
@RequestParam(required = false) String board, |
||||
|
@RequestParam(required = false) String topic, |
||||
|
final Pageable pageable) { |
||||
|
|
||||
|
return topicStarterService |
||||
|
.getWithFilters(user, board, topic, pageable) |
||||
|
.onFailure(throwable -> log.error("An error has occurred while processing a GET request", throwable)) |
||||
|
.map(ResponseEntity::ok) |
||||
|
.get(); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,6 @@ |
|||||
|
spring: |
||||
|
datasource: |
||||
|
driver-class-name: org.postgresql.Driver |
||||
|
url: jdbc:postgresql://localhost:5432/${POSTGRES_DB}?ApplicationName=topic-starters |
||||
|
username: ${POSTGRES_USER} |
||||
|
password: ${POSTGRES_PASSWORD} |
||||
@ -0,0 +1,42 @@ |
|||||
|
server: |
||||
|
port: 8080 |
||||
|
compression: |
||||
|
enabled: true |
||||
|
min-response-size: 1024 |
||||
|
mime-types: application/json,application/xml,text/plain |
||||
|
|
||||
|
spring: |
||||
|
datasource: |
||||
|
driver-class-name: org.postgresql.Driver |
||||
|
url: jdbc:postgresql://topic-starters-postgres:5432/${POSTGRES_DB}?ApplicationName=topic-starters |
||||
|
username: ${POSTGRES_USER} |
||||
|
password: ${POSTGRES_PASSWORD} |
||||
|
sql-script-encoding: UTF-8 |
||||
|
initialization-mode: never |
||||
|
|
||||
|
jpa: |
||||
|
hibernate: |
||||
|
ddl-auto: validate |
||||
|
naming: |
||||
|
implicit-strategy: org.hibernate.boot.model.naming.ImplicitNamingStrategyComponentPathImpl |
||||
|
|
||||
|
properties: |
||||
|
hibernate: |
||||
|
format_sql: true |
||||
|
show_sql: false |
||||
|
jdbc: |
||||
|
lob: |
||||
|
non_contextual_creation: true |
||||
|
|
||||
|
flyway: |
||||
|
enabled: true |
||||
|
|
||||
|
servlet: |
||||
|
multipart: |
||||
|
enabled: true |
||||
|
file-size-threshold: 2KB |
||||
|
max-file-size: 128MB |
||||
|
max-request-size: 256MB |
||||
|
|
||||
|
config: |
||||
|
additional-location: classpath:/config/development/ |
||||
@ -0,0 +1,42 @@ |
|||||
|
version: '3.2' |
||||
|
|
||||
|
services: |
||||
|
topic_starters_postgres_data: |
||||
|
image: postgres:10.7 |
||||
|
container_name: topic-starters-postgres |
||||
|
expose: |
||||
|
- "5432" |
||||
|
volumes: |
||||
|
- 'topic_starters_postgres_data:/var/lib/postgresql/data' |
||||
|
env_file: |
||||
|
- env/topic_starters_postgres.env |
||||
|
ports: |
||||
|
- "5432:5432" |
||||
|
networks: |
||||
|
- topic-starters-net |
||||
|
restart: on-failure |
||||
|
|
||||
|
topic_starters: |
||||
|
build: ./ |
||||
|
container_name: topic-starters-service |
||||
|
ports: |
||||
|
- "8080:8080" |
||||
|
env_file: |
||||
|
- ./env/topic_starters_postgres.env |
||||
|
secrets: |
||||
|
- topic_starters_username |
||||
|
- topic_starters_password |
||||
|
networks: |
||||
|
- topic-starters-net |
||||
|
restart: on-failure |
||||
|
|
||||
|
volumes: |
||||
|
topic_starters_postgres_data: |
||||
|
networks: |
||||
|
topic-starters-net: |
||||
|
driver: bridge |
||||
|
secrets: |
||||
|
topic_starters_username: |
||||
|
file: ./secrets/username |
||||
|
topic_starters_password: |
||||
|
file: ./secrets/password |
||||
@ -0,0 +1,83 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<parent> |
||||
|
<artifactId>topicstarter</artifactId> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<version>1.0.0</version> |
||||
|
</parent> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
|
||||
|
<artifactId>entity</artifactId> |
||||
|
|
||||
|
<dependencies> |
||||
|
<!-- Data Access Dependencies --> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter-data-jpa</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>com.querydsl</groupId> |
||||
|
<artifactId>querydsl-jpa</artifactId> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Database migration dependencies --> |
||||
|
<dependency> |
||||
|
<groupId>org.flywaydb</groupId> |
||||
|
<artifactId>flyway-core</artifactId> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Utility Dependencies--> |
||||
|
<dependency> |
||||
|
<groupId>org.projectlombok</groupId> |
||||
|
<artifactId>lombok</artifactId> |
||||
|
<scope>provided</scope> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
|
||||
|
<build> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>com.mysema.maven</groupId> |
||||
|
<artifactId>apt-maven-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<goals> |
||||
|
<goal>process</goal> |
||||
|
</goals> |
||||
|
<configuration> |
||||
|
<outputDirectory>target/generated-sources/java</outputDirectory> |
||||
|
<processor>com.querydsl.apt.jpa.JPAAnnotationProcessor</processor> |
||||
|
</configuration> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-source-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-sources</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-javadoc-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-javadocs</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
</project> |
||||
@ -0,0 +1,23 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.entity; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
|
||||
|
import javax.persistence.Column; |
||||
|
import javax.persistence.Id; |
||||
|
import javax.persistence.MappedSuperclass; |
||||
|
import javax.persistence.PrePersist; |
||||
|
import java.util.UUID; |
||||
|
|
||||
|
@Data |
||||
|
@MappedSuperclass |
||||
|
public abstract class AbstractEntity { |
||||
|
|
||||
|
@Id |
||||
|
@Column(columnDefinition = "UUID") |
||||
|
protected String id; |
||||
|
|
||||
|
@PrePersist |
||||
|
protected void onCreate() { |
||||
|
this.id = UUID.randomUUID().toString(); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,9 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.entity; |
||||
|
|
||||
|
import org.springframework.boot.autoconfigure.domain.EntityScan; |
||||
|
import org.springframework.context.annotation.Configuration; |
||||
|
|
||||
|
@Configuration |
||||
|
@EntityScan |
||||
|
public class TopicStarterEntityConfiguration { |
||||
|
} |
||||
@ -0,0 +1,47 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.entity.topic; |
||||
|
|
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.AbstractEntity; |
||||
|
import lombok.Data; |
||||
|
import lombok.EqualsAndHashCode; |
||||
|
|
||||
|
import javax.persistence.Column; |
||||
|
import javax.persistence.Entity; |
||||
|
import javax.persistence.Table; |
||||
|
|
||||
|
@Data |
||||
|
@EqualsAndHashCode(callSuper = true) |
||||
|
@Entity |
||||
|
@Table(name = "topic_starter") |
||||
|
public class TopicStarter extends AbstractEntity { |
||||
|
|
||||
|
@Column(nullable = false) |
||||
|
private Long topicId; |
||||
|
|
||||
|
@Column(nullable = false) |
||||
|
private String topicUrl; |
||||
|
|
||||
|
@Column(nullable = false) |
||||
|
private String starterUsername; |
||||
|
|
||||
|
private String starterUrl; |
||||
|
|
||||
|
private Long starterId; |
||||
|
|
||||
|
@Column(nullable = false) |
||||
|
private String boardTitle; |
||||
|
|
||||
|
@Column(nullable = false) |
||||
|
private String boardUrl; |
||||
|
|
||||
|
@Column(nullable = false) |
||||
|
private Long boardId; |
||||
|
|
||||
|
@Column(nullable = false) |
||||
|
private String topicSubject; |
||||
|
|
||||
|
@Column(nullable = false) |
||||
|
private Long numberOfReplies; |
||||
|
|
||||
|
@Column(nullable = false) |
||||
|
private Long numberOfViews; |
||||
|
} |
||||
@ -0,0 +1,16 @@ |
|||||
|
create table topic_starter |
||||
|
( |
||||
|
id varchar(255) not null |
||||
|
constraint topic_starter_id_pkey primary key, |
||||
|
topic_id bigint not null, |
||||
|
topic_url varchar(255) not null, |
||||
|
starter_username varchar(255) not null, |
||||
|
starter_url varchar(255) not null, |
||||
|
starter_id bigint not null, |
||||
|
board_title varchar(255) not null, |
||||
|
board_url varchar(255) not null, |
||||
|
board_id bigint not null, |
||||
|
topic_subject varchar(255) not null, |
||||
|
number_of_replies bigint not null, |
||||
|
number_of_views bigint not null |
||||
|
); |
||||
@ -0,0 +1,3 @@ |
|||||
|
POSTGRES_USER=postgres |
||||
|
POSTGRES_PASSWORD=postgres |
||||
|
POSTGRES_DB=topic-starters |
||||
@ -0,0 +1,201 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?> |
||||
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
|
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>topicstarter</artifactId> |
||||
|
<version>1.0.0</version> |
||||
|
<packaging>pom</packaging> |
||||
|
|
||||
|
<parent> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter-parent</artifactId> |
||||
|
<version>2.1.6.RELEASE</version> |
||||
|
</parent> |
||||
|
|
||||
|
<modules> |
||||
|
<module>api</module> |
||||
|
<module>service</module> |
||||
|
<module>entity</module> |
||||
|
<module>repository</module> |
||||
|
<module>scheduled</module> |
||||
|
</modules> |
||||
|
|
||||
|
<properties> |
||||
|
<buildNumber/> |
||||
|
<excludePattern/> |
||||
|
|
||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
||||
|
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> |
||||
|
|
||||
|
<java.version>1.8</java.version> |
||||
|
|
||||
|
<maven-source-plugin.version>3.0.1</maven-source-plugin.version> |
||||
|
<maven-javadoc-plugin.version>3.0.1</maven-javadoc-plugin.version> |
||||
|
<maven-compiler-plugin.version>3.5.1</maven-compiler-plugin.version> |
||||
|
<apt-maven-plugin.version>1.1.3</apt-maven-plugin.version> |
||||
|
|
||||
|
<vavr.version>1.0.0-alpha-3</vavr.version> |
||||
|
<lombok.version>1.18.8</lombok.version> |
||||
|
<jsoup.version>1.12.1</jsoup.version> |
||||
|
<okhttp.version>3.9.0</okhttp.version> |
||||
|
<jackson.version>2.9.0</jackson.version> |
||||
|
</properties> |
||||
|
|
||||
|
<dependencyManagement> |
||||
|
<dependencies> |
||||
|
<!-- App modules --> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>api</artifactId> |
||||
|
<version>${project.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>entity</artifactId> |
||||
|
<version>${project.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>repository</artifactId> |
||||
|
<version>${project.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>scheduled</artifactId> |
||||
|
<version>${project.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>service</artifactId> |
||||
|
<version>${project.version}</version> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Utils --> |
||||
|
<dependency> |
||||
|
<groupId>io.vavr</groupId> |
||||
|
<artifactId>vavr</artifactId> |
||||
|
<version>${vavr.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.projectlombok</groupId> |
||||
|
<artifactId>lombok</artifactId> |
||||
|
<version>${lombok.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.jsoup</groupId> |
||||
|
<artifactId>jsoup</artifactId> |
||||
|
<version>${jsoup.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>com.squareup.okhttp3</groupId> |
||||
|
<artifactId>okhttp</artifactId> |
||||
|
<version>${okhttp.version}</version> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- DB --> |
||||
|
<dependency> |
||||
|
<groupId>com.querydsl</groupId> |
||||
|
<artifactId>querydsl-jpa</artifactId> |
||||
|
<version>${querydsl.version}</version> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Other --> |
||||
|
<dependency> |
||||
|
<groupId>com.fasterxml.jackson.core</groupId> |
||||
|
<artifactId>jackson-annotations</artifactId> |
||||
|
<version>${jackson.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>com.fasterxml.jackson.datatype</groupId> |
||||
|
<artifactId>jackson-datatype-jsr310</artifactId> |
||||
|
<version>${jackson.version}</version> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
</dependencyManagement> |
||||
|
|
||||
|
<build> |
||||
|
<pluginManagement> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-compiler-plugin</artifactId> |
||||
|
<version>${maven-compiler-plugin.version}</version> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>com.mysema.maven</groupId> |
||||
|
<artifactId>apt-maven-plugin</artifactId> |
||||
|
<version>${apt-maven-plugin.version}</version> |
||||
|
<dependencies> |
||||
|
<dependency> |
||||
|
<groupId>com.querydsl</groupId> |
||||
|
<artifactId>querydsl-apt</artifactId> |
||||
|
<version>${querydsl.version}</version> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-source-plugin</artifactId> |
||||
|
<version>${maven-source-plugin.version}</version> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-javadoc-plugin</artifactId> |
||||
|
<version>${maven-javadoc-plugin.version}</version> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</pluginManagement> |
||||
|
</build> |
||||
|
|
||||
|
<repositories> |
||||
|
<repository> |
||||
|
<id>spring-milestones</id> |
||||
|
<name>Spring Milestones</name> |
||||
|
<url>https://repo.spring.io/milestone</url> |
||||
|
<snapshots> |
||||
|
<enabled>false</enabled> |
||||
|
</snapshots> |
||||
|
</repository> |
||||
|
<repository> |
||||
|
<id>spring-snapshots</id> |
||||
|
<name>Spring Snapshots</name> |
||||
|
<url>https://repo.spring.io/snapshot</url> |
||||
|
<snapshots> |
||||
|
</snapshots> |
||||
|
</repository> |
||||
|
|
||||
|
<repository> |
||||
|
<id>jitpack.io</id> |
||||
|
<url>https://jitpack.io</url> |
||||
|
</repository> |
||||
|
</repositories> |
||||
|
|
||||
|
<pluginRepositories> |
||||
|
<pluginRepository> |
||||
|
<snapshots> |
||||
|
<enabled>false</enabled> |
||||
|
</snapshots> |
||||
|
<id>jcenter-central</id> |
||||
|
<name>bintray-plugins</name> |
||||
|
<url>https://jcenter.bintray.com</url> |
||||
|
</pluginRepository> |
||||
|
|
||||
|
<pluginRepository> |
||||
|
<id>spring-milestones</id> |
||||
|
<name>Spring Milestones</name> |
||||
|
<url>https://repo.spring.io/milestone</url> |
||||
|
<snapshots> |
||||
|
<enabled>false</enabled> |
||||
|
</snapshots> |
||||
|
</pluginRepository> |
||||
|
<pluginRepository> |
||||
|
<id>spring-snapshots</id> |
||||
|
<name>Spring Snapshots</name> |
||||
|
<url>https://repo.spring.io/snapshot</url> |
||||
|
<snapshots> |
||||
|
</snapshots> |
||||
|
</pluginRepository> |
||||
|
</pluginRepositories> |
||||
|
</project> |
||||
@ -0,0 +1,74 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<parent> |
||||
|
<artifactId>topicstarter</artifactId> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<version>1.0.0</version> |
||||
|
</parent> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
|
||||
|
<artifactId>repository</artifactId> |
||||
|
|
||||
|
<dependencies> |
||||
|
<!-- Module Dependencies --> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>entity</artifactId> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Data Querying Dependencies --> |
||||
|
<dependency> |
||||
|
<groupId>com.querydsl</groupId> |
||||
|
<artifactId>querydsl-apt</artifactId> |
||||
|
<scope>provided</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>com.querydsl</groupId> |
||||
|
<artifactId>querydsl-jpa</artifactId> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Utility Dependencies--> |
||||
|
|
||||
|
<dependency> |
||||
|
<groupId>org.projectlombok</groupId> |
||||
|
<artifactId>lombok</artifactId> |
||||
|
<scope>provided</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>io.vavr</groupId> |
||||
|
<artifactId>vavr</artifactId> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
|
||||
|
<build> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-source-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-sources</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-javadoc-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-javadocs</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
</project> |
||||
@ -0,0 +1,11 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter; |
||||
|
|
||||
|
import org.springframework.data.jpa.repository.JpaRepository; |
||||
|
import org.springframework.data.querydsl.QuerydslPredicateExecutor; |
||||
|
import org.springframework.data.repository.NoRepositoryBean; |
||||
|
|
||||
|
import java.util.UUID; |
||||
|
|
||||
|
@NoRepositoryBean |
||||
|
public interface GenericRepository<T> extends JpaRepository<T, UUID>, QuerydslPredicateExecutor<T> { |
||||
|
} |
||||
@ -0,0 +1,11 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter; |
||||
|
|
||||
|
import org.springframework.context.annotation.ComponentScan; |
||||
|
import org.springframework.context.annotation.Configuration; |
||||
|
import org.springframework.data.jpa.repository.config.EnableJpaRepositories; |
||||
|
|
||||
|
@Configuration |
||||
|
@ComponentScan |
||||
|
@EnableJpaRepositories |
||||
|
public class TopicStarterRepositoryConfig { |
||||
|
} |
||||
@ -0,0 +1,9 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.repository.topic; |
||||
|
|
||||
|
import gr.thmmy.mthmmy.topicstarter.GenericRepository; |
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
||||
|
import org.springframework.stereotype.Repository; |
||||
|
|
||||
|
@Repository |
||||
|
public interface TopicStarterRepository extends GenericRepository<TopicStarter> { |
||||
|
} |
||||
@ -0,0 +1,8 @@ |
|||||
|
#!/bin/sh |
||||
|
|
||||
|
topic_starters_username=$(cat /run/secrets/topic_starters_username) |
||||
|
topic_starters_password=$(cat /run/secrets/topic_starters_password) |
||||
|
|
||||
|
java -DTOPIC_STARTERS_USERNAME="$topic_starters_username" \ |
||||
|
-DTOPIC_STARTERS_PASSWORD="$topic_starters_password" \ |
||||
|
-jar app.jar gr.thmmy.mthmmy.topicstarter.api.TopicStarterApplication |
||||
@ -0,0 +1,89 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
|
||||
|
<parent> |
||||
|
<artifactId>topicstarter</artifactId> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<version>1.0.0</version> |
||||
|
</parent> |
||||
|
|
||||
|
<artifactId>scheduled</artifactId> |
||||
|
|
||||
|
<dependencies> |
||||
|
<!-- Module Dependencies --> |
||||
|
|
||||
|
|
||||
|
<!-- Basic Spring Dependencies --> |
||||
|
|
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-configuration-processor</artifactId> |
||||
|
<optional>true</optional> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Utility Dependencies--> |
||||
|
|
||||
|
<dependency> |
||||
|
<groupId>org.projectlombok</groupId> |
||||
|
<artifactId>lombok</artifactId> |
||||
|
<scope>provided</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>io.vavr</groupId> |
||||
|
<artifactId>vavr</artifactId> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Template Compiler --> |
||||
|
|
||||
|
<dependency> |
||||
|
<groupId>com.github.spullara.mustache.java</groupId> |
||||
|
<artifactId>compiler</artifactId> |
||||
|
<version>0.9.5</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>entity</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>service</artifactId> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
|
||||
|
<build> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-source-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-sources</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-javadoc-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-javadocs</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
</project> |
||||
@ -0,0 +1,36 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.scheduled; |
||||
|
|
||||
|
import gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.TopicStarterParserService; |
||||
|
import lombok.Data; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.scheduling.annotation.Scheduled; |
||||
|
import org.springframework.stereotype.Component; |
||||
|
|
||||
|
import javax.annotation.PostConstruct; |
||||
|
|
||||
|
@Data |
||||
|
@Slf4j |
||||
|
@Component |
||||
|
public class TopicStarterScheduled { |
||||
|
|
||||
|
private final TopicStarterParserService topicStarterParserService; |
||||
|
|
||||
|
// Runs at 02:00am every day of every month
|
||||
|
@Scheduled(cron = "0 0 02 * * *") |
||||
|
public void run() throws Exception { |
||||
|
|
||||
|
topicStarterParserService |
||||
|
.parseTopicStarters() |
||||
|
.onFailure(throwable -> log.error("An error has occurred while processing a GET request", throwable)) |
||||
|
.get(); |
||||
|
} |
||||
|
|
||||
|
@PostConstruct |
||||
|
public void init() { |
||||
|
|
||||
|
topicStarterParserService |
||||
|
.parseTopicStarters() |
||||
|
.onFailure(throwable -> log.error("An error has occurred while processing a GET request", throwable)) |
||||
|
.get(); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,11 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.scheduled; |
||||
|
|
||||
|
import org.springframework.context.annotation.ComponentScan; |
||||
|
import org.springframework.context.annotation.Configuration; |
||||
|
import org.springframework.scheduling.annotation.EnableScheduling; |
||||
|
|
||||
|
@Configuration |
||||
|
@ComponentScan |
||||
|
@EnableScheduling |
||||
|
public class TopicStarterSchedulesConfig { |
||||
|
} |
||||
@ -0,0 +1,93 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
|
||||
|
<parent> |
||||
|
<artifactId>topicstarter</artifactId> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<version>1.0.0</version> |
||||
|
</parent> |
||||
|
|
||||
|
<artifactId>service</artifactId> |
||||
|
|
||||
|
<dependencies> |
||||
|
<!-- Module Dependencies --> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>entity</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
||||
|
<artifactId>repository</artifactId> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Basic Spring Dependencies --> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-configuration-processor</artifactId> |
||||
|
<optional>true</optional> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Utility Dependencies--> |
||||
|
<dependency> |
||||
|
<groupId>org.projectlombok</groupId> |
||||
|
<artifactId>lombok</artifactId> |
||||
|
<scope>provided</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>io.vavr</groupId> |
||||
|
<artifactId>vavr</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.jsoup</groupId> |
||||
|
<artifactId>jsoup</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>com.squareup.okhttp3</groupId> |
||||
|
<artifactId>okhttp</artifactId> |
||||
|
</dependency> |
||||
|
|
||||
|
<!-- Template Compiler --> |
||||
|
<dependency> |
||||
|
<groupId>com.github.spullara.mustache.java</groupId> |
||||
|
<artifactId>compiler</artifactId> |
||||
|
<version>0.9.5</version> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
|
||||
|
<build> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-source-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-sources</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-javadoc-plugin</artifactId> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-javadocs</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
</project> |
||||
@ -0,0 +1,46 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.service; |
||||
|
|
||||
|
import io.vavr.control.Try; |
||||
|
import okhttp3.Cookie; |
||||
|
import okhttp3.CookieJar; |
||||
|
import okhttp3.HttpUrl; |
||||
|
import okhttp3.OkHttpClient; |
||||
|
import org.springframework.context.annotation.Bean; |
||||
|
import org.springframework.context.annotation.ComponentScan; |
||||
|
import org.springframework.context.annotation.Configuration; |
||||
|
import org.springframework.lang.NonNull; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
|
||||
|
@Configuration |
||||
|
@ComponentScan |
||||
|
public class TopicStarterServiceConfiguration { |
||||
|
|
||||
|
@Bean |
||||
|
public OkHttpClient getClient() { |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> new CookieJar() { |
||||
|
private final java.util.List<Cookie> cookieStore = new ArrayList<>(); |
||||
|
|
||||
|
@Override |
||||
|
public void saveFromResponse(@NonNull HttpUrl url, @NonNull java.util.List<Cookie> cookies) { |
||||
|
cookieStore.addAll(cookies); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public java.util.List<Cookie> loadForRequest(@NonNull HttpUrl url) { |
||||
|
return cookieStore; |
||||
|
} |
||||
|
} |
||||
|
).map(cookieJar -> new OkHttpClient.Builder() |
||||
|
.cookieJar(cookieJar) |
||||
|
.connectTimeout(30, TimeUnit.SECONDS) |
||||
|
.writeTimeout(30, TimeUnit.SECONDS) |
||||
|
.readTimeout(30, TimeUnit.SECONDS) |
||||
|
.retryOnConnectionFailure(true) |
||||
|
.build()) |
||||
|
.get(); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,14 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.service.topic.starter; |
||||
|
|
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
||||
|
import io.vavr.control.Try; |
||||
|
import org.springframework.data.domain.Page; |
||||
|
import org.springframework.data.domain.Pageable; |
||||
|
|
||||
|
public interface TopicStarterService { |
||||
|
|
||||
|
Try<Page<TopicStarter>> getWithFilters(String user, |
||||
|
String board, |
||||
|
String topic, |
||||
|
Pageable pageable); |
||||
|
} |
||||
@ -0,0 +1,116 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.service.topic.starter; |
||||
|
|
||||
|
import com.querydsl.core.types.dsl.BooleanExpression; |
||||
|
import com.querydsl.core.types.dsl.Expressions; |
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.topic.QTopicStarter; |
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
||||
|
import gr.thmmy.mthmmy.topicstarter.repository.topic.TopicStarterRepository; |
||||
|
import io.vavr.control.Option; |
||||
|
import io.vavr.control.Try; |
||||
|
import lombok.Data; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import org.springframework.data.domain.Page; |
||||
|
import org.springframework.data.domain.Pageable; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
|
||||
|
import javax.annotation.Nonnull; |
||||
|
import javax.annotation.Nullable; |
||||
|
|
||||
|
import static java.util.Objects.requireNonNull; |
||||
|
|
||||
|
@RequiredArgsConstructor |
||||
|
@Data |
||||
|
@Service |
||||
|
public class TopicStarterServiceImpl implements TopicStarterService { |
||||
|
|
||||
|
private final TopicStarterRepository topicStarterRepository; |
||||
|
|
||||
|
@Override |
||||
|
public Try<Page<TopicStarter>> getWithFilters(final @Nullable String user, |
||||
|
final @Nullable String board, |
||||
|
final @Nullable String topic, |
||||
|
final Pageable pageable) { |
||||
|
requireNonNull(pageable, "pageable is null"); |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> Expressions.asBoolean(true).isTrue()) |
||||
|
.flatMap(topicStarterPredicate -> Option |
||||
|
.of(user) |
||||
|
.map(userNotNull -> extractId(userNotNull) |
||||
|
.toTry() |
||||
|
.flatMap(this::getUserIdPredicate) |
||||
|
.orElse(getUsernamePredicate(userNotNull)) |
||||
|
.map(topicStarterPredicate::and)) |
||||
|
.getOrElse(Try.success(topicStarterPredicate)) |
||||
|
) |
||||
|
.flatMap(topicStarterPredicate -> Option |
||||
|
.of(board) |
||||
|
.map(boardNotNull -> extractId(boardNotNull) |
||||
|
.toTry() |
||||
|
.flatMap(this::getBoardIdPredicate) |
||||
|
.orElse(getBoardTitlePredicate(boardNotNull)) |
||||
|
.map(topicStarterPredicate::and)) |
||||
|
.getOrElse(Try.success(topicStarterPredicate)) |
||||
|
) |
||||
|
.flatMap(topicStarterPredicate -> Option |
||||
|
.of(topic) |
||||
|
.map(topicNotNull -> extractId(topicNotNull) |
||||
|
.toTry() |
||||
|
.flatMap(this::getTopicIdPredicate) |
||||
|
.orElse(getTopicSubjectPredicate(topicNotNull)) |
||||
|
.map(topicStarterPredicate::and)) |
||||
|
.getOrElse(Try.success(topicStarterPredicate)) |
||||
|
) |
||||
|
.map(topicStarterPredicate -> topicStarterRepository.findAll(topicStarterPredicate, pageable)); |
||||
|
} |
||||
|
|
||||
|
private Option<Long> extractId(final @Nonnull String input) { |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> Long.parseLong(input)) |
||||
|
.recoverWith(throwable -> Try.success(null)) |
||||
|
.toOption(); |
||||
|
} |
||||
|
|
||||
|
private Try<BooleanExpression> getUsernamePredicate(final @Nonnull String username) { |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> QTopicStarter.topicStarter) |
||||
|
.map(qTopicStarter -> qTopicStarter.starterUsername.like(username)); |
||||
|
} |
||||
|
|
||||
|
private Try<BooleanExpression> getUserIdPredicate(final @Nonnull Long userId) { |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> QTopicStarter.topicStarter) |
||||
|
.map(qTopicStarter -> qTopicStarter.starterId.eq(userId)); |
||||
|
} |
||||
|
|
||||
|
private Try<BooleanExpression> getBoardTitlePredicate(final @Nonnull String boardTitle) { |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> QTopicStarter.topicStarter) |
||||
|
.map(qTopicStarter -> qTopicStarter.boardTitle.like(boardTitle)); |
||||
|
} |
||||
|
|
||||
|
private Try<BooleanExpression> getBoardIdPredicate(final @Nonnull Long boardId) { |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> QTopicStarter.topicStarter) |
||||
|
.map(qTopicStarter -> qTopicStarter.boardId.eq(boardId)); |
||||
|
} |
||||
|
|
||||
|
private Try<BooleanExpression> getTopicSubjectPredicate(final @Nonnull String topicSubject) { |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> QTopicStarter.topicStarter) |
||||
|
.map(qTopicStarter -> qTopicStarter.topicSubject.like(topicSubject)); |
||||
|
} |
||||
|
|
||||
|
private Try<BooleanExpression> getTopicIdPredicate(final @Nonnull Long topicId) { |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> QTopicStarter.topicStarter) |
||||
|
.map(qTopicStarter -> qTopicStarter.topicId.eq(topicId)); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,8 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser; |
||||
|
|
||||
|
import io.vavr.control.Try; |
||||
|
|
||||
|
public interface TopicStarterParserService { |
||||
|
|
||||
|
Try<Void> parseTopicStarters(); |
||||
|
} |
||||
@ -0,0 +1,241 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser; |
||||
|
|
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.topic.QTopicStarter; |
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
||||
|
import gr.thmmy.mthmmy.topicstarter.repository.topic.TopicStarterRepository; |
||||
|
import gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util.TopicParserUtils; |
||||
|
import io.vavr.Tuple; |
||||
|
import io.vavr.collection.List; |
||||
|
import io.vavr.control.Option; |
||||
|
import io.vavr.control.Try; |
||||
|
import lombok.Data; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import okhttp3.*; |
||||
|
import org.jsoup.Jsoup; |
||||
|
import org.jsoup.nodes.Document; |
||||
|
import org.jsoup.nodes.Element; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.core.env.Environment; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
|
||||
|
import java.util.stream.IntStream; |
||||
|
|
||||
|
import static gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util.BoardParserUtils.extractBoardIdFromUrl; |
||||
|
import static gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util.BoardParserUtils.parseSubBoards; |
||||
|
import static java.util.Objects.requireNonNull; |
||||
|
|
||||
|
@Service |
||||
|
@Data |
||||
|
@Slf4j |
||||
|
public class TopicStarterParserServiceImpl implements TopicStarterParserService { |
||||
|
/* Constants */ |
||||
|
private static final String baseUrl = "https://www.thmmy.gr/smf/index.php?action=forum"; |
||||
|
private static final String RECYCLING_BIN_BOARD_ID = "244.0"; |
||||
|
private static final String USERNAME_ENV_VAR = "TOPIC_STARTERS_USERNAME"; |
||||
|
private static final String PASSWORD_ENV_VAR = "TOPIC_STARTERS_PASSWORD"; |
||||
|
private static final HttpUrl loginUrl = HttpUrl.parse("https://www.thmmy.gr/smf/index.php?action=login2"); |
||||
|
|
||||
|
private final OkHttpClient client; |
||||
|
private final TopicStarterRepository topicStarterRepository; |
||||
|
|
||||
|
@Autowired |
||||
|
private Environment environment; |
||||
|
|
||||
|
@Override |
||||
|
public Try<Void> parseTopicStarters() { |
||||
|
|
||||
|
return login() |
||||
|
.flatMap(ignored -> parseBoard(baseUrl)); |
||||
|
} |
||||
|
|
||||
|
private Try<Void> parseBoard(final String url) { |
||||
|
requireNonNull(url, "url is null"); |
||||
|
|
||||
|
return Try.of(Request.Builder::new) |
||||
|
// Builds and executes request
|
||||
|
.map(requestBuilder -> requestBuilder |
||||
|
.url(url) |
||||
|
.build()) |
||||
|
.mapTry(request -> client |
||||
|
.newCall(request) |
||||
|
.execute()) |
||||
|
.flatMap(this::getResponseString) |
||||
|
.map(Jsoup::parse) |
||||
|
.flatMap(document -> parseSubBoards(document) // Parses the sub boards
|
||||
|
.flatMap(subBoards -> Option |
||||
|
.of(subBoards) |
||||
|
.toTry() |
||||
|
.flatMap(subBoardsNotNull -> Try |
||||
|
.run(() -> subBoardsNotNull |
||||
|
.stream() |
||||
|
.filter(subBoard -> !subBoard |
||||
|
.attr("href") |
||||
|
.contains("board=" + RECYCLING_BIN_BOARD_ID)) |
||||
|
.forEach(subBoard -> parseBoard(subBoard.attr("href"))) |
||||
|
) |
||||
|
) |
||||
|
) |
||||
|
// Parses the topics
|
||||
|
.flatMap(ignored -> parseTopics(document, url)) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
private Try<Void> parseTopics(final Document document, |
||||
|
final String boardUrl) { |
||||
|
requireNonNull(document, "document is null"); |
||||
|
requireNonNull(boardUrl, "boardUrl is null"); |
||||
|
|
||||
|
return Try.of(() -> document // Finds the number of pages in this board
|
||||
|
.select("a.navPages") |
||||
|
.last()) |
||||
|
.map(pageNumber -> Option |
||||
|
.of(pageNumber) |
||||
|
.map(Element::text) |
||||
|
.map(Integer::parseInt) |
||||
|
.getOrElse(1)) |
||||
|
.flatMap(numberOfPages -> Try // Parses the board title
|
||||
|
.of(() -> document |
||||
|
.select("div.nav>b>a") |
||||
|
.last() |
||||
|
.text()) |
||||
|
.flatMap(boardTitle -> extractBoardIdFromUrl(boardUrl) // Parses topics of the current page
|
||||
|
.flatMap(boardId -> saveTopics(document, boardTitle, boardUrl, boardId) |
||||
|
.flatMap(ignored -> IntStream // Parses the topics from the rest of the pages
|
||||
|
.range(1, numberOfPages) |
||||
|
.boxed() |
||||
|
.map(page -> Try // Builds the URL of the board for each page
|
||||
|
.of(() -> String.join(".", |
||||
|
boardUrl.substring(0, boardUrl.lastIndexOf(".")), |
||||
|
String.valueOf(page * 20)) |
||||
|
).flatMap(pageUrl -> Try |
||||
|
.of(Request.Builder::new) |
||||
|
.map(requestBuilder -> requestBuilder |
||||
|
.url(pageUrl) |
||||
|
.build() |
||||
|
) |
||||
|
) |
||||
|
.mapTry(request -> client |
||||
|
.newCall(request) |
||||
|
.execute()) |
||||
|
.flatMap(this::getResponseString) |
||||
|
.map(Jsoup::parse) |
||||
|
.flatMap(pageDocument -> saveTopics( |
||||
|
pageDocument, |
||||
|
boardTitle, |
||||
|
boardUrl, |
||||
|
boardId))) |
||||
|
.collect(List.collector()) |
||||
|
.transform(Try::sequence) |
||||
|
) |
||||
|
) |
||||
|
).map(ignored -> null)); |
||||
|
} |
||||
|
|
||||
|
private Try<Void> saveTopics(final Document document, |
||||
|
final String boardTitle, |
||||
|
final String boardUrl, |
||||
|
final Long boardId) { |
||||
|
requireNonNull(document, "document is null"); |
||||
|
requireNonNull(boardTitle, "boardTitle is null"); |
||||
|
requireNonNull(boardUrl, "boardUrl is null"); |
||||
|
requireNonNull(boardId, "boardId is null"); |
||||
|
|
||||
|
return Try // Finds this page's topics
|
||||
|
.of(() -> document |
||||
|
.select("table.bordercolor tbody>tr:not([class])")) |
||||
|
.flatMap(topics -> Try |
||||
|
.run(() -> topics |
||||
|
.forEach(topicRow -> TopicParserUtils |
||||
|
.parseTopic(topicRow, boardTitle, boardUrl, boardId) |
||||
|
.map(topicStarter -> savedTopicStarter(topicStarter) |
||||
|
.flatMap(savedTopicStarter -> updateSavedTopicStarter(savedTopicStarter, topicStarter)) |
||||
|
.getOrElse(topicStarter) |
||||
|
) |
||||
|
.map(topicStarterRepository::save) |
||||
|
) |
||||
|
) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
private Option<TopicStarter> savedTopicStarter(final TopicStarter topicStarter) { |
||||
|
requireNonNull(topicStarter, "topicStarter is null"); |
||||
|
|
||||
|
return Option |
||||
|
.of(QTopicStarter.topicStarter) |
||||
|
.map(qTopicStarter -> qTopicStarter.topicId.eq(topicStarter.getTopicId())) |
||||
|
.map(topicStarterRepository::findOne) |
||||
|
.flatMap(Option::ofOptional); |
||||
|
} |
||||
|
|
||||
|
private Option<TopicStarter> updateSavedTopicStarter(final TopicStarter savedTopicStarter, final TopicStarter newTopicStarter) { |
||||
|
requireNonNull(savedTopicStarter, "savedTopicStarter is null"); |
||||
|
requireNonNull(newTopicStarter, "newTopicStarter is null"); |
||||
|
|
||||
|
savedTopicStarter.setTopicId(newTopicStarter.getTopicId()); |
||||
|
savedTopicStarter.setTopicUrl(newTopicStarter.getTopicUrl()); |
||||
|
savedTopicStarter.setStarterUsername(newTopicStarter.getStarterUsername()); |
||||
|
savedTopicStarter.setStarterUrl(newTopicStarter.getStarterUrl()); |
||||
|
savedTopicStarter.setStarterId(newTopicStarter.getStarterId()); |
||||
|
savedTopicStarter.setBoardTitle(newTopicStarter.getBoardTitle()); |
||||
|
savedTopicStarter.setBoardUrl(newTopicStarter.getBoardUrl()); |
||||
|
savedTopicStarter.setBoardId(newTopicStarter.getBoardId()); |
||||
|
savedTopicStarter.setTopicSubject(newTopicStarter.getTopicSubject()); |
||||
|
savedTopicStarter.setNumberOfReplies(newTopicStarter.getNumberOfReplies()); |
||||
|
savedTopicStarter.setNumberOfViews(newTopicStarter.getNumberOfViews()); |
||||
|
|
||||
|
return Option.of(savedTopicStarter); |
||||
|
} |
||||
|
|
||||
|
private Try<Void> login() { |
||||
|
|
||||
|
return Option |
||||
|
.of(environment.getProperty(USERNAME_ENV_VAR)) |
||||
|
.map(username -> Option |
||||
|
.of(environment.getProperty(PASSWORD_ENV_VAR)) |
||||
|
.map(password -> Tuple.of(username, password)) |
||||
|
.getOrElseThrow(() -> new RuntimeException("Password is null")) |
||||
|
).map(loginSecrets -> Option |
||||
|
.of(loginUrl) |
||||
|
.map(loginUrlNotNull -> Try |
||||
|
.of(FormBody.Builder::new) |
||||
|
.map(builder -> builder |
||||
|
.add("user", loginSecrets._1) |
||||
|
.add("passwrd", loginSecrets._2) |
||||
|
.add("cookielength", "-1") // -1 is forever
|
||||
|
.build() |
||||
|
).flatMap(formBody -> Try |
||||
|
.of(Request.Builder::new) |
||||
|
.map(builder -> builder |
||||
|
.url(loginUrlNotNull) |
||||
|
.post(formBody) |
||||
|
.build() |
||||
|
) |
||||
|
).mapTry(request -> client |
||||
|
.newCall(request) |
||||
|
.execute() |
||||
|
).flatMap(response -> Try |
||||
|
.run(() -> response |
||||
|
.body() |
||||
|
.close() |
||||
|
) |
||||
|
) |
||||
|
).getOrElseThrow(() -> new RuntimeException("Login URL is null.")) |
||||
|
).getOrElseThrow(() -> new RuntimeException("Username is null")); |
||||
|
} |
||||
|
|
||||
|
private Try<String> getResponseString(final Response response) { |
||||
|
requireNonNull(response, "response is null"); |
||||
|
|
||||
|
// Checks response for null and closes response body
|
||||
|
return Try |
||||
|
.of(response::body) |
||||
|
.flatMap(responseBody -> Option |
||||
|
.of(responseBody) |
||||
|
.toTry() |
||||
|
.mapTry(ResponseBody::string) |
||||
|
.flatMap(responseBodyString -> Try |
||||
|
.run(responseBody::close) |
||||
|
.map(ignored -> responseBodyString)) |
||||
|
); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,33 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util; |
||||
|
|
||||
|
import io.vavr.control.Try; |
||||
|
import org.jsoup.nodes.Document; |
||||
|
import org.jsoup.select.Elements; |
||||
|
|
||||
|
import java.util.regex.Pattern; |
||||
|
|
||||
|
import static java.util.Objects.requireNonNull; |
||||
|
|
||||
|
public abstract class BoardParserUtils { |
||||
|
|
||||
|
public static Try<Long> extractBoardIdFromUrl(final String url) { |
||||
|
requireNonNull(url, "url is null"); |
||||
|
|
||||
|
return Try.success(".+?board=([0-9]+)") |
||||
|
.map(regex -> Pattern.compile(regex, Pattern.MULTILINE)) |
||||
|
.map(pattern -> pattern.matcher(url)) |
||||
|
.map(matcher -> Try |
||||
|
.of(matcher::find) |
||||
|
.filter(aBoolean -> aBoolean) |
||||
|
.map(ignored -> Long.parseLong(matcher.group(1))) |
||||
|
.getOrElse(-1L) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
public static Try<Elements> parseSubBoards(final Document document) { |
||||
|
requireNonNull(document, "document is null"); |
||||
|
|
||||
|
return Try.of(() -> document |
||||
|
.select("div.tborder tbody tr.windowbg2 td>b>a[name^=b]")); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,177 @@ |
|||||
|
package gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util; |
||||
|
|
||||
|
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
||||
|
import io.vavr.control.Option; |
||||
|
import io.vavr.control.Try; |
||||
|
import org.jsoup.nodes.Element; |
||||
|
import org.jsoup.select.Elements; |
||||
|
|
||||
|
import javax.annotation.Nullable; |
||||
|
import java.util.regex.Pattern; |
||||
|
|
||||
|
import static java.util.Objects.requireNonNull; |
||||
|
|
||||
|
public abstract class TopicParserUtils { |
||||
|
|
||||
|
public static Try<TopicStarter> parseTopic(final Element topicRow, |
||||
|
final String boardTitle, |
||||
|
final String boardUrl, |
||||
|
final Long boardId) { |
||||
|
requireNonNull(topicRow, "topicRow is null"); |
||||
|
requireNonNull(boardTitle, "boardTitle is null"); |
||||
|
requireNonNull(boardUrl, "boardUrl is null"); |
||||
|
requireNonNull(boardId, "boardId is null"); |
||||
|
|
||||
|
return Try.of(() -> topicRow |
||||
|
.select("td")) |
||||
|
.flatMap(topicColumns -> Try |
||||
|
.of(() -> topicColumns |
||||
|
.get(3) |
||||
|
.select("a") |
||||
|
.first()) |
||||
|
.flatMap(starterUrlElement -> Try |
||||
|
.of(TopicStarter::new) |
||||
|
.flatMap(topic -> parseTopicSubject(topicColumns) |
||||
|
.map(topicSubject -> { |
||||
|
topic.setTopicSubject(topicSubject); |
||||
|
|
||||
|
return topic; |
||||
|
}) |
||||
|
).flatMap(topic -> parseTopicUrl(topicColumns) |
||||
|
.flatMap(topicUrl -> extractTopicIdFromUrl(topicUrl) |
||||
|
.map(topicId -> { |
||||
|
topic.setTopicUrl(topicUrl); |
||||
|
topic.setTopicId(topicId); |
||||
|
|
||||
|
return topic; |
||||
|
}) |
||||
|
) |
||||
|
).flatMap(topic -> parseTopicStarterUsername(topicColumns) |
||||
|
.map(starterUsername -> { |
||||
|
topic.setStarterUsername(starterUsername); |
||||
|
|
||||
|
return topic; |
||||
|
}) |
||||
|
).flatMap(topic -> parseTopicStarterUrl(starterUrlElement) |
||||
|
.flatMap(topicStarterUrl -> extractTopicStarterIdFromUrl(topicStarterUrl) |
||||
|
.map(topicStarterId -> { |
||||
|
topic.setStarterUrl(topicStarterUrl); |
||||
|
topic.setStarterId(topicStarterId); |
||||
|
|
||||
|
return topic; |
||||
|
}) |
||||
|
) |
||||
|
).flatMap(topic -> parseTopicNumberOfReplies(topicColumns) |
||||
|
.map(numReplies -> { |
||||
|
topic.setNumberOfReplies(numReplies); |
||||
|
|
||||
|
return topic; |
||||
|
}) |
||||
|
).flatMap(topic -> parseTopicNumberOfViews(topicColumns) |
||||
|
.map(numViews -> { |
||||
|
topic.setNumberOfViews(numViews); |
||||
|
|
||||
|
return topic; |
||||
|
}) |
||||
|
).map(topic -> { |
||||
|
topic.setBoardTitle(boardTitle); |
||||
|
topic.setBoardId(boardId); |
||||
|
topic.setBoardUrl(boardUrl); |
||||
|
|
||||
|
return topic; |
||||
|
} |
||||
|
) |
||||
|
) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
private static Try<String> parseTopicSubject(final Elements topicColumns) { |
||||
|
requireNonNull(topicColumns, "topicColumns is null"); |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> topicColumns |
||||
|
.get(2) |
||||
|
.select("span>a") |
||||
|
.first() |
||||
|
.text()); |
||||
|
} |
||||
|
|
||||
|
private static Try<String> parseTopicUrl(final Elements topicColumns) { |
||||
|
requireNonNull(topicColumns, "topicColumns is null"); |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> topicColumns |
||||
|
.get(2) |
||||
|
.select("span>a") |
||||
|
.first() |
||||
|
.attr("href")); |
||||
|
} |
||||
|
|
||||
|
private static Try<String> parseTopicStarterUsername(final Elements topicColumns) { |
||||
|
requireNonNull(topicColumns, "topicColumns is null"); |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> topicColumns |
||||
|
.get(3) |
||||
|
.text()); |
||||
|
} |
||||
|
|
||||
|
private static Try<String> parseTopicStarterUrl(final Element starterUrlEl) { |
||||
|
requireNonNull(starterUrlEl, "starterUrlEl is null"); |
||||
|
|
||||
|
return Try.of(() -> Option |
||||
|
.of(starterUrlEl) |
||||
|
.map(starterUrlElNotNull -> starterUrlElNotNull |
||||
|
.attr("href")) |
||||
|
.getOrElse(() -> null)); |
||||
|
} |
||||
|
|
||||
|
private static Try<Long> parseTopicNumberOfReplies(final Elements topicColumns) { |
||||
|
requireNonNull(topicColumns, "topicColumns is null"); |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> topicColumns |
||||
|
.get(4) |
||||
|
.text()) |
||||
|
.map(Long::parseLong); |
||||
|
} |
||||
|
|
||||
|
private static Try<Long> parseTopicNumberOfViews(final Elements topicColumns) { |
||||
|
requireNonNull(topicColumns, "topicColumns is null"); |
||||
|
|
||||
|
return Try |
||||
|
.of(() -> topicColumns |
||||
|
.get(5) |
||||
|
.text()) |
||||
|
.map(Long::parseLong); |
||||
|
} |
||||
|
|
||||
|
private static Try<Long> extractTopicIdFromUrl(final String topicUrl) { |
||||
|
requireNonNull(topicUrl, "topicUrl is null"); |
||||
|
|
||||
|
return Try.success(".+?topic=([0-9]+)") |
||||
|
.map(regex -> Pattern.compile(regex, Pattern.MULTILINE)) |
||||
|
.map(pattern -> pattern.matcher(topicUrl)) |
||||
|
.map(matcher -> Try.of(matcher::find) |
||||
|
.filter(aBoolean -> aBoolean) |
||||
|
.map(ignored -> Long.parseLong(matcher.group(1))) |
||||
|
.getOrElse(-1L) |
||||
|
); |
||||
|
|
||||
|
} |
||||
|
|
||||
|
private static Try<Long> extractTopicStarterIdFromUrl(final @Nullable String topicStarterUrl) { |
||||
|
|
||||
|
return Option |
||||
|
.of(topicStarterUrl) |
||||
|
.map(topicStarterUrlNotNull -> Try |
||||
|
.of(() -> ".+?profile;u=([0-9]+)") |
||||
|
.map(regex -> Pattern.compile(regex, Pattern.MULTILINE)) |
||||
|
.map(pattern -> pattern.matcher(topicStarterUrlNotNull)) |
||||
|
.map(matcher -> Try.of(matcher::find) |
||||
|
.filter(aBoolean -> aBoolean) |
||||
|
.map(ignored -> Long.parseLong(matcher.group(1))) |
||||
|
.getOrElse(-1L)) |
||||
|
).getOrElse(Try.success(-1L)); |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue