commit
7d9e806d75
35 changed files with 1897 additions and 0 deletions
@ -0,0 +1,131 @@ |
|||
### JetBrains template |
|||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm |
|||
|
|||
*.iml |
|||
|
|||
## Directory-based project format: |
|||
.idea/ |
|||
/out/ |
|||
|
|||
|
|||
### NetBeans template |
|||
nbproject/private/ |
|||
build/ |
|||
nbbuild/ |
|||
dist/ |
|||
nbdist/ |
|||
nbactions.xml |
|||
nb-configuration.xml |
|||
.nb-gradle/ |
|||
|
|||
|
|||
### Eclipse template |
|||
*.pydevproject |
|||
.metadata |
|||
.gradle |
|||
bin/ |
|||
tmp/ |
|||
*.tmp |
|||
*.bak |
|||
*.swp |
|||
*~.nib |
|||
local.properties |
|||
.settings/ |
|||
.loadpath |
|||
|
|||
# Eclipse Core |
|||
.project |
|||
|
|||
# External tool builders |
|||
.externalToolBuilders/ |
|||
|
|||
# Locally stored "Eclipse launch configurations" |
|||
*.launch |
|||
|
|||
# CDT-specific |
|||
.cproject |
|||
|
|||
# JDT-specific (Eclipse Java Development Tools) |
|||
.classpath |
|||
|
|||
# PDT-specific |
|||
.buildpath |
|||
|
|||
# sbteclipse plugin |
|||
.target |
|||
|
|||
# TeXlipse plugin |
|||
.texlipse |
|||
|
|||
|
|||
### Maven template |
|||
target/ |
|||
pom.xml.tag |
|||
pom.xml.releaseBackup |
|||
pom.xml.versionsBackup |
|||
pom.xml.next |
|||
release.properties |
|||
dependency-reduced-pom.xml |
|||
buildNumber.properties |
|||
.mvn/timing.properties |
|||
|
|||
|
|||
### Linux template |
|||
*~ |
|||
|
|||
# KDE directory preferences |
|||
.directory |
|||
|
|||
# Linux trash folder which might appear on any partition or disk |
|||
.Trash-* |
|||
|
|||
|
|||
### Windows template |
|||
# Windows image file caches |
|||
Thumbs.db |
|||
ehthumbs.db |
|||
|
|||
# Folder config file |
|||
Desktop.ini |
|||
|
|||
# Recycle Bin used on file shares |
|||
$RECYCLE.BIN/ |
|||
|
|||
# Windows Installer files |
|||
*.cab |
|||
*.msi |
|||
*.msm |
|||
*.msp |
|||
|
|||
# Windows shortcuts |
|||
*.lnk |
|||
|
|||
|
|||
### OSX template |
|||
.DS_Store |
|||
.AppleDouble |
|||
.LSOverride |
|||
|
|||
# Icon must end with two \r |
|||
Icon |
|||
|
|||
# Thumbnails |
|||
._* |
|||
|
|||
# Files that might appear in the root of a volume |
|||
.DocumentRevisions-V100 |
|||
.fseventsd |
|||
.Spotlight-V100 |
|||
.TemporaryItems |
|||
.Trashes |
|||
.VolumeIcon.icns |
|||
|
|||
# Directories potentially created on remote AFP share |
|||
.AppleDB |
|||
.AppleDesktop |
|||
Network Trash Folder |
|||
Temporary Items |
|||
.apdisk |
|||
|
|||
/secrets |
|||
/dump* |
|||
@ -0,0 +1,10 @@ |
|||
FROM openjdk:8-jre-alpine as build |
|||
WORKDIR /workspace/app |
|||
|
|||
WORKDIR /topic-starters-app |
|||
COPY ./api/target/topicstarters-api.jar app.jar |
|||
COPY ./run.sh run.sh |
|||
|
|||
RUN chmod +x run.sh |
|||
|
|||
ENTRYPOINT ["./run.sh"] |
|||
@ -0,0 +1,10 @@ |
|||
build: |
|||
@docker-compose -p topicstarters build; |
|||
run: |
|||
@docker-compose -p topicstarters up -d |
|||
stop: |
|||
@docker-compose -p topicstarters down |
|||
clean-data: |
|||
@docker-compose -p topicstarters down -v |
|||
clean-images: |
|||
@docker rmi `docker images -q -f "dangling=true"` |
|||
@ -0,0 +1,107 @@ |
|||
# Thmmy Topic Starters |
|||
> A service that parses all the topics of thmmy.gr into a database and exposes an endpoint for getting filtered pages of them |
|||
|
|||
Thmmy topic starters is an application that crawls all thmmy.gr boards every day at 2 a.m. Information parsed are then saved in a |
|||
postgres database and can be accessed by the endpoint `/api/topicstarters`. |
|||
|
|||
--- |
|||
|
|||
# API endpoint |
|||
|
|||
## View topic starters |
|||
|
|||
``` |
|||
GET /api/topicstarters |
|||
``` |
|||
|
|||
### Parameters |
|||
|
|||
| Name | Type | Description | |
|||
| ----- | ------ | ----------- | |
|||
| user | String | **Optional**. The username or ID of the user. Filters the results by user. | |
|||
| board | String | **Optional**. The title or ID of the board. Filters the results by board. | |
|||
| topic | String | **Optional**. The subject or ID of the topic. Filters the results by topic. | |
|||
|
|||
#### Example |
|||
|
|||
```shell script |
|||
curl --location \ |
|||
--request GET 'localhost:8080/api/topicstarters' \ |
|||
--form 'user=14670' \ |
|||
--form 'board=Ανακοινώσεις και Έκτακτα νέα' \ |
|||
--form 'topic=68000' |
|||
``` |
|||
|
|||
### Response |
|||
|
|||
``` |
|||
Status: 200 OK |
|||
Content-Type: application/json;charset=UTF-8 |
|||
Content-Length: 962 |
|||
Content-Encoding: gzip |
|||
``` |
|||
```json |
|||
{ |
|||
"content": [ |
|||
{ |
|||
"id": "d806599f-ae77-4780-bd3d-510943588054", |
|||
"topicId": 68000, |
|||
"topicUrl": "https://www.thmmy.gr/smf/index.php?topic=68000.0", |
|||
"starterUsername": "Apostolof", |
|||
"starterUrl": "https://www.thmmy.gr/smf/index.php?action=profile;u=14670", |
|||
"starterId": 14670, |
|||
"boardTitle": "Ανακοινώσεις και Έκτακτα νέα", |
|||
"boardUrl": "https://www.thmmy.gr/smf/index.php?board=25.0", |
|||
"boardId": 25, |
|||
"topicSubject": "mTHMMY (alpha version)", |
|||
"numberOfReplies": 175, |
|||
"numberOfViews": 15729 |
|||
} |
|||
], |
|||
"pageable": { |
|||
[...] |
|||
}, |
|||
[...] |
|||
} |
|||
``` |
|||
|
|||
\* part of the response truncated for brevity |
|||
|
|||
--- |
|||
|
|||
# Build docker image |
|||
|
|||
To build the docker image you first need to build the java application for production: |
|||
```shell script |
|||
mvn clean install package |
|||
``` |
|||
|
|||
Define a username, password and database name for the postgres database in the file `./env/topic_starters_postgres.env`. |
|||
An example of what this file might look like is given in `./env/topic_starters_postgres.example.env`. |
|||
|
|||
If you want to get all the topics accessible by a logged-in user (rather that just those publicly available to guests) you also need to create two more files containing the username and password of a user for the application to use. |
|||
* `./secrets/username`: which should contain the username |
|||
* `./secrets/password`: which should contain the password |
|||
|
|||
Then just use the Makefile to handle the build: |
|||
```shell script |
|||
make build |
|||
``` |
|||
|
|||
Run the image using: |
|||
```shell script |
|||
make run |
|||
``` |
|||
|
|||
Stop the container using: |
|||
```shell script |
|||
make stop |
|||
``` |
|||
|
|||
The Makefile also provides targets for cleaning the data and dangling images. |
|||
|
|||
--- |
|||
|
|||
## License |
|||
|
|||
[](https://gitlab.com/Apostolof/flavours-without-borders/blob/master/LICENSE.md) |
|||
@ -0,0 +1,103 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
|||
<parent> |
|||
<artifactId>topicstarter</artifactId> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<version>1.0.0</version> |
|||
</parent> |
|||
<modelVersion>4.0.0</modelVersion> |
|||
|
|||
<artifactId>api</artifactId> |
|||
|
|||
<properties> |
|||
<start-class>gr.thmmy.mthmmy.topicstarter.api.TopicStarterApplication</start-class> |
|||
</properties> |
|||
|
|||
<dependencies> |
|||
<!-- Module Dependencies --> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>entity</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>repository</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>scheduled</artifactId> |
|||
<scope>compile</scope> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>service</artifactId> |
|||
</dependency> |
|||
|
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-configuration-processor</artifactId> |
|||
<optional>true</optional> |
|||
</dependency> |
|||
|
|||
<!-- Web Dependencies --> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter-web</artifactId> |
|||
</dependency> |
|||
|
|||
<!-- Utility Dependencies--> |
|||
<dependency> |
|||
<groupId>org.projectlombok</groupId> |
|||
<artifactId>lombok</artifactId> |
|||
<scope>provided</scope> |
|||
</dependency> |
|||
|
|||
<!-- Database Dependencies --> |
|||
<dependency> |
|||
<groupId>org.postgresql</groupId> |
|||
<artifactId>postgresql</artifactId> |
|||
<scope>runtime</scope> |
|||
</dependency> |
|||
</dependencies> |
|||
|
|||
<build> |
|||
<plugins> |
|||
<plugin> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-maven-plugin</artifactId> |
|||
<configuration> |
|||
<finalName>topicstarters</finalName> |
|||
<classifier>api</classifier> |
|||
<executable>true</executable> |
|||
</configuration> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-source-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-sources</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-javadoc-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-javadocs</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
</plugins> |
|||
</build> |
|||
</project> |
|||
@ -0,0 +1,18 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.api; |
|||
|
|||
import gr.thmmy.mthmmy.topicstarter.TopicStarterRepositoryConfig; |
|||
import gr.thmmy.mthmmy.topicstarter.entity.TopicStarterEntityConfiguration; |
|||
import gr.thmmy.mthmmy.topicstarter.scheduled.TopicStarterSchedulesConfig; |
|||
import gr.thmmy.mthmmy.topicstarter.service.TopicStarterServiceConfiguration; |
|||
import org.springframework.context.annotation.Configuration; |
|||
import org.springframework.context.annotation.Import; |
|||
|
|||
@Configuration |
|||
@Import({ |
|||
TopicStarterServiceConfiguration.class, |
|||
TopicStarterRepositoryConfig.class, |
|||
TopicStarterEntityConfiguration.class, |
|||
TopicStarterSchedulesConfig.class |
|||
}) |
|||
public class TopicStarterApiConfig { |
|||
} |
|||
@ -0,0 +1,19 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.api; |
|||
|
|||
import org.springframework.boot.SpringApplication; |
|||
import org.springframework.boot.autoconfigure.SpringBootApplication; |
|||
import org.springframework.boot.builder.SpringApplicationBuilder; |
|||
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; |
|||
|
|||
@SpringBootApplication |
|||
public class TopicStarterApplication extends SpringBootServletInitializer { |
|||
|
|||
public static void main(final String... args) { |
|||
SpringApplication.run(TopicStarterApplication.class, args); |
|||
} |
|||
|
|||
@Override |
|||
protected SpringApplicationBuilder configure(final SpringApplicationBuilder application) { |
|||
return application.sources(TopicStarterApplication.class); |
|||
} |
|||
} |
|||
@ -0,0 +1,14 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.api.controller.topic.starter; |
|||
|
|||
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
|||
import org.springframework.data.domain.Page; |
|||
import org.springframework.data.domain.Pageable; |
|||
import org.springframework.http.ResponseEntity; |
|||
|
|||
public interface TopicStarterController { |
|||
|
|||
ResponseEntity<Page<TopicStarter>> topics(String user, |
|||
String board, |
|||
String topic, |
|||
Pageable pageable); |
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.api.controller.topic.starter; |
|||
|
|||
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
|||
import gr.thmmy.mthmmy.topicstarter.service.topic.starter.TopicStarterService; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.data.domain.Page; |
|||
import org.springframework.data.domain.Pageable; |
|||
import org.springframework.http.ResponseEntity; |
|||
import org.springframework.web.bind.annotation.GetMapping; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RequestParam; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
|
|||
@RestController |
|||
@RequestMapping("/api") |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
public class TopicStarterControllerImpl implements TopicStarterController { |
|||
|
|||
private final TopicStarterService topicStarterService; |
|||
|
|||
@Override |
|||
@GetMapping("/topicstarters") |
|||
public ResponseEntity<Page<TopicStarter>> topics(@RequestParam(required = false) String user, |
|||
@RequestParam(required = false) String board, |
|||
@RequestParam(required = false) String topic, |
|||
final Pageable pageable) { |
|||
|
|||
return topicStarterService |
|||
.getWithFilters(user, board, topic, pageable) |
|||
.onFailure(throwable -> log.error("An error has occurred while processing a GET request", throwable)) |
|||
.map(ResponseEntity::ok) |
|||
.get(); |
|||
} |
|||
} |
|||
@ -0,0 +1,6 @@ |
|||
spring: |
|||
datasource: |
|||
driver-class-name: org.postgresql.Driver |
|||
url: jdbc:postgresql://localhost:5432/${POSTGRES_DB}?ApplicationName=topic-starters |
|||
username: ${POSTGRES_USER} |
|||
password: ${POSTGRES_PASSWORD} |
|||
@ -0,0 +1,42 @@ |
|||
server: |
|||
port: 8080 |
|||
compression: |
|||
enabled: true |
|||
min-response-size: 1024 |
|||
mime-types: application/json,application/xml,text/plain |
|||
|
|||
spring: |
|||
datasource: |
|||
driver-class-name: org.postgresql.Driver |
|||
url: jdbc:postgresql://topic-starters-postgres:5432/${POSTGRES_DB}?ApplicationName=topic-starters |
|||
username: ${POSTGRES_USER} |
|||
password: ${POSTGRES_PASSWORD} |
|||
sql-script-encoding: UTF-8 |
|||
initialization-mode: never |
|||
|
|||
jpa: |
|||
hibernate: |
|||
ddl-auto: validate |
|||
naming: |
|||
implicit-strategy: org.hibernate.boot.model.naming.ImplicitNamingStrategyComponentPathImpl |
|||
|
|||
properties: |
|||
hibernate: |
|||
format_sql: true |
|||
show_sql: false |
|||
jdbc: |
|||
lob: |
|||
non_contextual_creation: true |
|||
|
|||
flyway: |
|||
enabled: true |
|||
|
|||
servlet: |
|||
multipart: |
|||
enabled: true |
|||
file-size-threshold: 2KB |
|||
max-file-size: 128MB |
|||
max-request-size: 256MB |
|||
|
|||
config: |
|||
additional-location: classpath:/config/development/ |
|||
@ -0,0 +1,42 @@ |
|||
version: '3.2' |
|||
|
|||
services: |
|||
topic_starters_postgres_data: |
|||
image: postgres:10.7 |
|||
container_name: topic-starters-postgres |
|||
expose: |
|||
- "5432" |
|||
volumes: |
|||
- 'topic_starters_postgres_data:/var/lib/postgresql/data' |
|||
env_file: |
|||
- env/topic_starters_postgres.env |
|||
ports: |
|||
- "5432:5432" |
|||
networks: |
|||
- topic-starters-net |
|||
restart: on-failure |
|||
|
|||
topic_starters: |
|||
build: ./ |
|||
container_name: topic-starters-service |
|||
ports: |
|||
- "8080:8080" |
|||
env_file: |
|||
- ./env/topic_starters_postgres.env |
|||
secrets: |
|||
- topic_starters_username |
|||
- topic_starters_password |
|||
networks: |
|||
- topic-starters-net |
|||
restart: on-failure |
|||
|
|||
volumes: |
|||
topic_starters_postgres_data: |
|||
networks: |
|||
topic-starters-net: |
|||
driver: bridge |
|||
secrets: |
|||
topic_starters_username: |
|||
file: ./secrets/username |
|||
topic_starters_password: |
|||
file: ./secrets/password |
|||
@ -0,0 +1,83 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
|||
<parent> |
|||
<artifactId>topicstarter</artifactId> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<version>1.0.0</version> |
|||
</parent> |
|||
<modelVersion>4.0.0</modelVersion> |
|||
|
|||
<artifactId>entity</artifactId> |
|||
|
|||
<dependencies> |
|||
<!-- Data Access Dependencies --> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter-data-jpa</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>com.querydsl</groupId> |
|||
<artifactId>querydsl-jpa</artifactId> |
|||
</dependency> |
|||
|
|||
<!-- Database migration dependencies --> |
|||
<dependency> |
|||
<groupId>org.flywaydb</groupId> |
|||
<artifactId>flyway-core</artifactId> |
|||
</dependency> |
|||
|
|||
<!-- Utility Dependencies--> |
|||
<dependency> |
|||
<groupId>org.projectlombok</groupId> |
|||
<artifactId>lombok</artifactId> |
|||
<scope>provided</scope> |
|||
</dependency> |
|||
</dependencies> |
|||
|
|||
<build> |
|||
<plugins> |
|||
<plugin> |
|||
<groupId>com.mysema.maven</groupId> |
|||
<artifactId>apt-maven-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<goals> |
|||
<goal>process</goal> |
|||
</goals> |
|||
<configuration> |
|||
<outputDirectory>target/generated-sources/java</outputDirectory> |
|||
<processor>com.querydsl.apt.jpa.JPAAnnotationProcessor</processor> |
|||
</configuration> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-source-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-sources</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-javadoc-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-javadocs</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
</plugins> |
|||
</build> |
|||
</project> |
|||
@ -0,0 +1,23 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.entity; |
|||
|
|||
import lombok.Data; |
|||
|
|||
import javax.persistence.Column; |
|||
import javax.persistence.Id; |
|||
import javax.persistence.MappedSuperclass; |
|||
import javax.persistence.PrePersist; |
|||
import java.util.UUID; |
|||
|
|||
@Data |
|||
@MappedSuperclass |
|||
public abstract class AbstractEntity { |
|||
|
|||
@Id |
|||
@Column(columnDefinition = "UUID") |
|||
protected String id; |
|||
|
|||
@PrePersist |
|||
protected void onCreate() { |
|||
this.id = UUID.randomUUID().toString(); |
|||
} |
|||
} |
|||
@ -0,0 +1,9 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.entity; |
|||
|
|||
import org.springframework.boot.autoconfigure.domain.EntityScan; |
|||
import org.springframework.context.annotation.Configuration; |
|||
|
|||
@Configuration |
|||
@EntityScan |
|||
public class TopicStarterEntityConfiguration { |
|||
} |
|||
@ -0,0 +1,47 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.entity.topic; |
|||
|
|||
import gr.thmmy.mthmmy.topicstarter.entity.AbstractEntity; |
|||
import lombok.Data; |
|||
import lombok.EqualsAndHashCode; |
|||
|
|||
import javax.persistence.Column; |
|||
import javax.persistence.Entity; |
|||
import javax.persistence.Table; |
|||
|
|||
@Data |
|||
@EqualsAndHashCode(callSuper = true) |
|||
@Entity |
|||
@Table(name = "topic_starter") |
|||
public class TopicStarter extends AbstractEntity { |
|||
|
|||
@Column(nullable = false) |
|||
private Long topicId; |
|||
|
|||
@Column(nullable = false) |
|||
private String topicUrl; |
|||
|
|||
@Column(nullable = false) |
|||
private String starterUsername; |
|||
|
|||
private String starterUrl; |
|||
|
|||
private Long starterId; |
|||
|
|||
@Column(nullable = false) |
|||
private String boardTitle; |
|||
|
|||
@Column(nullable = false) |
|||
private String boardUrl; |
|||
|
|||
@Column(nullable = false) |
|||
private Long boardId; |
|||
|
|||
@Column(nullable = false) |
|||
private String topicSubject; |
|||
|
|||
@Column(nullable = false) |
|||
private Long numberOfReplies; |
|||
|
|||
@Column(nullable = false) |
|||
private Long numberOfViews; |
|||
} |
|||
@ -0,0 +1,16 @@ |
|||
create table topic_starter |
|||
( |
|||
id varchar(255) not null |
|||
constraint topic_starter_id_pkey primary key, |
|||
topic_id bigint not null, |
|||
topic_url varchar(255) not null, |
|||
starter_username varchar(255) not null, |
|||
starter_url varchar(255) not null, |
|||
starter_id bigint not null, |
|||
board_title varchar(255) not null, |
|||
board_url varchar(255) not null, |
|||
board_id bigint not null, |
|||
topic_subject varchar(255) not null, |
|||
number_of_replies bigint not null, |
|||
number_of_views bigint not null |
|||
); |
|||
@ -0,0 +1,3 @@ |
|||
POSTGRES_USER=postgres |
|||
POSTGRES_PASSWORD=postgres |
|||
POSTGRES_DB=topic-starters |
|||
@ -0,0 +1,201 @@ |
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?> |
|||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
|||
<modelVersion>4.0.0</modelVersion> |
|||
|
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>topicstarter</artifactId> |
|||
<version>1.0.0</version> |
|||
<packaging>pom</packaging> |
|||
|
|||
<parent> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter-parent</artifactId> |
|||
<version>2.1.6.RELEASE</version> |
|||
</parent> |
|||
|
|||
<modules> |
|||
<module>api</module> |
|||
<module>service</module> |
|||
<module>entity</module> |
|||
<module>repository</module> |
|||
<module>scheduled</module> |
|||
</modules> |
|||
|
|||
<properties> |
|||
<buildNumber/> |
|||
<excludePattern/> |
|||
|
|||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
|||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> |
|||
|
|||
<java.version>1.8</java.version> |
|||
|
|||
<maven-source-plugin.version>3.0.1</maven-source-plugin.version> |
|||
<maven-javadoc-plugin.version>3.0.1</maven-javadoc-plugin.version> |
|||
<maven-compiler-plugin.version>3.5.1</maven-compiler-plugin.version> |
|||
<apt-maven-plugin.version>1.1.3</apt-maven-plugin.version> |
|||
|
|||
<vavr.version>1.0.0-alpha-3</vavr.version> |
|||
<lombok.version>1.18.8</lombok.version> |
|||
<jsoup.version>1.12.1</jsoup.version> |
|||
<okhttp.version>3.9.0</okhttp.version> |
|||
<jackson.version>2.9.0</jackson.version> |
|||
</properties> |
|||
|
|||
<dependencyManagement> |
|||
<dependencies> |
|||
<!-- App modules --> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>api</artifactId> |
|||
<version>${project.version}</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>entity</artifactId> |
|||
<version>${project.version}</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>repository</artifactId> |
|||
<version>${project.version}</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>scheduled</artifactId> |
|||
<version>${project.version}</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>service</artifactId> |
|||
<version>${project.version}</version> |
|||
</dependency> |
|||
|
|||
<!-- Utils --> |
|||
<dependency> |
|||
<groupId>io.vavr</groupId> |
|||
<artifactId>vavr</artifactId> |
|||
<version>${vavr.version}</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.projectlombok</groupId> |
|||
<artifactId>lombok</artifactId> |
|||
<version>${lombok.version}</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.jsoup</groupId> |
|||
<artifactId>jsoup</artifactId> |
|||
<version>${jsoup.version}</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>com.squareup.okhttp3</groupId> |
|||
<artifactId>okhttp</artifactId> |
|||
<version>${okhttp.version}</version> |
|||
</dependency> |
|||
|
|||
<!-- DB --> |
|||
<dependency> |
|||
<groupId>com.querydsl</groupId> |
|||
<artifactId>querydsl-jpa</artifactId> |
|||
<version>${querydsl.version}</version> |
|||
</dependency> |
|||
|
|||
<!-- Other --> |
|||
<dependency> |
|||
<groupId>com.fasterxml.jackson.core</groupId> |
|||
<artifactId>jackson-annotations</artifactId> |
|||
<version>${jackson.version}</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>com.fasterxml.jackson.datatype</groupId> |
|||
<artifactId>jackson-datatype-jsr310</artifactId> |
|||
<version>${jackson.version}</version> |
|||
</dependency> |
|||
</dependencies> |
|||
</dependencyManagement> |
|||
|
|||
<build> |
|||
<pluginManagement> |
|||
<plugins> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-compiler-plugin</artifactId> |
|||
<version>${maven-compiler-plugin.version}</version> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>com.mysema.maven</groupId> |
|||
<artifactId>apt-maven-plugin</artifactId> |
|||
<version>${apt-maven-plugin.version}</version> |
|||
<dependencies> |
|||
<dependency> |
|||
<groupId>com.querydsl</groupId> |
|||
<artifactId>querydsl-apt</artifactId> |
|||
<version>${querydsl.version}</version> |
|||
</dependency> |
|||
</dependencies> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-source-plugin</artifactId> |
|||
<version>${maven-source-plugin.version}</version> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-javadoc-plugin</artifactId> |
|||
<version>${maven-javadoc-plugin.version}</version> |
|||
</plugin> |
|||
</plugins> |
|||
</pluginManagement> |
|||
</build> |
|||
|
|||
<repositories> |
|||
<repository> |
|||
<id>spring-milestones</id> |
|||
<name>Spring Milestones</name> |
|||
<url>https://repo.spring.io/milestone</url> |
|||
<snapshots> |
|||
<enabled>false</enabled> |
|||
</snapshots> |
|||
</repository> |
|||
<repository> |
|||
<id>spring-snapshots</id> |
|||
<name>Spring Snapshots</name> |
|||
<url>https://repo.spring.io/snapshot</url> |
|||
<snapshots> |
|||
</snapshots> |
|||
</repository> |
|||
|
|||
<repository> |
|||
<id>jitpack.io</id> |
|||
<url>https://jitpack.io</url> |
|||
</repository> |
|||
</repositories> |
|||
|
|||
<pluginRepositories> |
|||
<pluginRepository> |
|||
<snapshots> |
|||
<enabled>false</enabled> |
|||
</snapshots> |
|||
<id>jcenter-central</id> |
|||
<name>bintray-plugins</name> |
|||
<url>https://jcenter.bintray.com</url> |
|||
</pluginRepository> |
|||
|
|||
<pluginRepository> |
|||
<id>spring-milestones</id> |
|||
<name>Spring Milestones</name> |
|||
<url>https://repo.spring.io/milestone</url> |
|||
<snapshots> |
|||
<enabled>false</enabled> |
|||
</snapshots> |
|||
</pluginRepository> |
|||
<pluginRepository> |
|||
<id>spring-snapshots</id> |
|||
<name>Spring Snapshots</name> |
|||
<url>https://repo.spring.io/snapshot</url> |
|||
<snapshots> |
|||
</snapshots> |
|||
</pluginRepository> |
|||
</pluginRepositories> |
|||
</project> |
|||
@ -0,0 +1,74 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
|||
<parent> |
|||
<artifactId>topicstarter</artifactId> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<version>1.0.0</version> |
|||
</parent> |
|||
<modelVersion>4.0.0</modelVersion> |
|||
|
|||
<artifactId>repository</artifactId> |
|||
|
|||
<dependencies> |
|||
<!-- Module Dependencies --> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>entity</artifactId> |
|||
</dependency> |
|||
|
|||
<!-- Data Querying Dependencies --> |
|||
<dependency> |
|||
<groupId>com.querydsl</groupId> |
|||
<artifactId>querydsl-apt</artifactId> |
|||
<scope>provided</scope> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>com.querydsl</groupId> |
|||
<artifactId>querydsl-jpa</artifactId> |
|||
</dependency> |
|||
|
|||
<!-- Utility Dependencies--> |
|||
|
|||
<dependency> |
|||
<groupId>org.projectlombok</groupId> |
|||
<artifactId>lombok</artifactId> |
|||
<scope>provided</scope> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>io.vavr</groupId> |
|||
<artifactId>vavr</artifactId> |
|||
</dependency> |
|||
</dependencies> |
|||
|
|||
<build> |
|||
<plugins> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-source-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-sources</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-javadoc-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-javadocs</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
</plugins> |
|||
</build> |
|||
</project> |
|||
@ -0,0 +1,11 @@ |
|||
package gr.thmmy.mthmmy.topicstarter; |
|||
|
|||
import org.springframework.data.jpa.repository.JpaRepository; |
|||
import org.springframework.data.querydsl.QuerydslPredicateExecutor; |
|||
import org.springframework.data.repository.NoRepositoryBean; |
|||
|
|||
import java.util.UUID; |
|||
|
|||
@NoRepositoryBean |
|||
public interface GenericRepository<T> extends JpaRepository<T, UUID>, QuerydslPredicateExecutor<T> { |
|||
} |
|||
@ -0,0 +1,11 @@ |
|||
package gr.thmmy.mthmmy.topicstarter; |
|||
|
|||
import org.springframework.context.annotation.ComponentScan; |
|||
import org.springframework.context.annotation.Configuration; |
|||
import org.springframework.data.jpa.repository.config.EnableJpaRepositories; |
|||
|
|||
@Configuration |
|||
@ComponentScan |
|||
@EnableJpaRepositories |
|||
public class TopicStarterRepositoryConfig { |
|||
} |
|||
@ -0,0 +1,9 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.repository.topic; |
|||
|
|||
import gr.thmmy.mthmmy.topicstarter.GenericRepository; |
|||
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
|||
import org.springframework.stereotype.Repository; |
|||
|
|||
@Repository |
|||
public interface TopicStarterRepository extends GenericRepository<TopicStarter> { |
|||
} |
|||
@ -0,0 +1,8 @@ |
|||
#!/bin/sh |
|||
|
|||
topic_starters_username=$(cat /run/secrets/topic_starters_username) |
|||
topic_starters_password=$(cat /run/secrets/topic_starters_password) |
|||
|
|||
java -DTOPIC_STARTERS_USERNAME="$topic_starters_username" \ |
|||
-DTOPIC_STARTERS_PASSWORD="$topic_starters_password" \ |
|||
-jar app.jar gr.thmmy.mthmmy.topicstarter.api.TopicStarterApplication |
|||
@ -0,0 +1,89 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
|||
<modelVersion>4.0.0</modelVersion> |
|||
|
|||
<parent> |
|||
<artifactId>topicstarter</artifactId> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<version>1.0.0</version> |
|||
</parent> |
|||
|
|||
<artifactId>scheduled</artifactId> |
|||
|
|||
<dependencies> |
|||
<!-- Module Dependencies --> |
|||
|
|||
|
|||
<!-- Basic Spring Dependencies --> |
|||
|
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-configuration-processor</artifactId> |
|||
<optional>true</optional> |
|||
</dependency> |
|||
|
|||
<!-- Utility Dependencies--> |
|||
|
|||
<dependency> |
|||
<groupId>org.projectlombok</groupId> |
|||
<artifactId>lombok</artifactId> |
|||
<scope>provided</scope> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>io.vavr</groupId> |
|||
<artifactId>vavr</artifactId> |
|||
</dependency> |
|||
|
|||
<!-- Template Compiler --> |
|||
|
|||
<dependency> |
|||
<groupId>com.github.spullara.mustache.java</groupId> |
|||
<artifactId>compiler</artifactId> |
|||
<version>0.9.5</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>entity</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>service</artifactId> |
|||
</dependency> |
|||
</dependencies> |
|||
|
|||
<build> |
|||
<plugins> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-source-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-sources</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-javadoc-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-javadocs</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
</plugins> |
|||
</build> |
|||
</project> |
|||
@ -0,0 +1,36 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.scheduled; |
|||
|
|||
import gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.TopicStarterParserService; |
|||
import lombok.Data; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.scheduling.annotation.Scheduled; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
import javax.annotation.PostConstruct; |
|||
|
|||
@Data |
|||
@Slf4j |
|||
@Component |
|||
public class TopicStarterScheduled { |
|||
|
|||
private final TopicStarterParserService topicStarterParserService; |
|||
|
|||
// Runs at 02:00am every day of every month
|
|||
@Scheduled(cron = "0 0 02 * * *") |
|||
public void run() throws Exception { |
|||
|
|||
topicStarterParserService |
|||
.parseTopicStarters() |
|||
.onFailure(throwable -> log.error("An error has occurred while processing a GET request", throwable)) |
|||
.get(); |
|||
} |
|||
|
|||
@PostConstruct |
|||
public void init() { |
|||
|
|||
topicStarterParserService |
|||
.parseTopicStarters() |
|||
.onFailure(throwable -> log.error("An error has occurred while processing a GET request", throwable)) |
|||
.get(); |
|||
} |
|||
} |
|||
@ -0,0 +1,11 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.scheduled; |
|||
|
|||
import org.springframework.context.annotation.ComponentScan; |
|||
import org.springframework.context.annotation.Configuration; |
|||
import org.springframework.scheduling.annotation.EnableScheduling; |
|||
|
|||
@Configuration |
|||
@ComponentScan |
|||
@EnableScheduling |
|||
public class TopicStarterSchedulesConfig { |
|||
} |
|||
@ -0,0 +1,93 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" |
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
|||
<modelVersion>4.0.0</modelVersion> |
|||
|
|||
<parent> |
|||
<artifactId>topicstarter</artifactId> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<version>1.0.0</version> |
|||
</parent> |
|||
|
|||
<artifactId>service</artifactId> |
|||
|
|||
<dependencies> |
|||
<!-- Module Dependencies --> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>entity</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>gr.thmmy.mthmmy.topicstarter</groupId> |
|||
<artifactId>repository</artifactId> |
|||
</dependency> |
|||
|
|||
<!-- Basic Spring Dependencies --> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-configuration-processor</artifactId> |
|||
<optional>true</optional> |
|||
</dependency> |
|||
|
|||
<!-- Utility Dependencies--> |
|||
<dependency> |
|||
<groupId>org.projectlombok</groupId> |
|||
<artifactId>lombok</artifactId> |
|||
<scope>provided</scope> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>io.vavr</groupId> |
|||
<artifactId>vavr</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.jsoup</groupId> |
|||
<artifactId>jsoup</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>com.squareup.okhttp3</groupId> |
|||
<artifactId>okhttp</artifactId> |
|||
</dependency> |
|||
|
|||
<!-- Template Compiler --> |
|||
<dependency> |
|||
<groupId>com.github.spullara.mustache.java</groupId> |
|||
<artifactId>compiler</artifactId> |
|||
<version>0.9.5</version> |
|||
</dependency> |
|||
</dependencies> |
|||
|
|||
<build> |
|||
<plugins> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-source-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-sources</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-javadoc-plugin</artifactId> |
|||
<executions> |
|||
<execution> |
|||
<id>attach-javadocs</id> |
|||
<phase>verify</phase> |
|||
<goals> |
|||
<goal>jar</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
</plugins> |
|||
</build> |
|||
</project> |
|||
@ -0,0 +1,46 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.service; |
|||
|
|||
import io.vavr.control.Try; |
|||
import okhttp3.Cookie; |
|||
import okhttp3.CookieJar; |
|||
import okhttp3.HttpUrl; |
|||
import okhttp3.OkHttpClient; |
|||
import org.springframework.context.annotation.Bean; |
|||
import org.springframework.context.annotation.ComponentScan; |
|||
import org.springframework.context.annotation.Configuration; |
|||
import org.springframework.lang.NonNull; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
@Configuration |
|||
@ComponentScan |
|||
public class TopicStarterServiceConfiguration { |
|||
|
|||
@Bean |
|||
public OkHttpClient getClient() { |
|||
|
|||
return Try |
|||
.of(() -> new CookieJar() { |
|||
private final java.util.List<Cookie> cookieStore = new ArrayList<>(); |
|||
|
|||
@Override |
|||
public void saveFromResponse(@NonNull HttpUrl url, @NonNull java.util.List<Cookie> cookies) { |
|||
cookieStore.addAll(cookies); |
|||
} |
|||
|
|||
@Override |
|||
public java.util.List<Cookie> loadForRequest(@NonNull HttpUrl url) { |
|||
return cookieStore; |
|||
} |
|||
} |
|||
).map(cookieJar -> new OkHttpClient.Builder() |
|||
.cookieJar(cookieJar) |
|||
.connectTimeout(30, TimeUnit.SECONDS) |
|||
.writeTimeout(30, TimeUnit.SECONDS) |
|||
.readTimeout(30, TimeUnit.SECONDS) |
|||
.retryOnConnectionFailure(true) |
|||
.build()) |
|||
.get(); |
|||
} |
|||
} |
|||
@ -0,0 +1,14 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.service.topic.starter; |
|||
|
|||
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
|||
import io.vavr.control.Try; |
|||
import org.springframework.data.domain.Page; |
|||
import org.springframework.data.domain.Pageable; |
|||
|
|||
public interface TopicStarterService { |
|||
|
|||
Try<Page<TopicStarter>> getWithFilters(String user, |
|||
String board, |
|||
String topic, |
|||
Pageable pageable); |
|||
} |
|||
@ -0,0 +1,116 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.service.topic.starter; |
|||
|
|||
import com.querydsl.core.types.dsl.BooleanExpression; |
|||
import com.querydsl.core.types.dsl.Expressions; |
|||
import gr.thmmy.mthmmy.topicstarter.entity.topic.QTopicStarter; |
|||
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
|||
import gr.thmmy.mthmmy.topicstarter.repository.topic.TopicStarterRepository; |
|||
import io.vavr.control.Option; |
|||
import io.vavr.control.Try; |
|||
import lombok.Data; |
|||
import lombok.RequiredArgsConstructor; |
|||
import org.springframework.data.domain.Page; |
|||
import org.springframework.data.domain.Pageable; |
|||
import org.springframework.stereotype.Service; |
|||
|
|||
import javax.annotation.Nonnull; |
|||
import javax.annotation.Nullable; |
|||
|
|||
import static java.util.Objects.requireNonNull; |
|||
|
|||
@RequiredArgsConstructor |
|||
@Data |
|||
@Service |
|||
public class TopicStarterServiceImpl implements TopicStarterService { |
|||
|
|||
private final TopicStarterRepository topicStarterRepository; |
|||
|
|||
@Override |
|||
public Try<Page<TopicStarter>> getWithFilters(final @Nullable String user, |
|||
final @Nullable String board, |
|||
final @Nullable String topic, |
|||
final Pageable pageable) { |
|||
requireNonNull(pageable, "pageable is null"); |
|||
|
|||
return Try |
|||
.of(() -> Expressions.asBoolean(true).isTrue()) |
|||
.flatMap(topicStarterPredicate -> Option |
|||
.of(user) |
|||
.map(userNotNull -> extractId(userNotNull) |
|||
.toTry() |
|||
.flatMap(this::getUserIdPredicate) |
|||
.orElse(getUsernamePredicate(userNotNull)) |
|||
.map(topicStarterPredicate::and)) |
|||
.getOrElse(Try.success(topicStarterPredicate)) |
|||
) |
|||
.flatMap(topicStarterPredicate -> Option |
|||
.of(board) |
|||
.map(boardNotNull -> extractId(boardNotNull) |
|||
.toTry() |
|||
.flatMap(this::getBoardIdPredicate) |
|||
.orElse(getBoardTitlePredicate(boardNotNull)) |
|||
.map(topicStarterPredicate::and)) |
|||
.getOrElse(Try.success(topicStarterPredicate)) |
|||
) |
|||
.flatMap(topicStarterPredicate -> Option |
|||
.of(topic) |
|||
.map(topicNotNull -> extractId(topicNotNull) |
|||
.toTry() |
|||
.flatMap(this::getTopicIdPredicate) |
|||
.orElse(getTopicSubjectPredicate(topicNotNull)) |
|||
.map(topicStarterPredicate::and)) |
|||
.getOrElse(Try.success(topicStarterPredicate)) |
|||
) |
|||
.map(topicStarterPredicate -> topicStarterRepository.findAll(topicStarterPredicate, pageable)); |
|||
} |
|||
|
|||
private Option<Long> extractId(final @Nonnull String input) { |
|||
|
|||
return Try |
|||
.of(() -> Long.parseLong(input)) |
|||
.recoverWith(throwable -> Try.success(null)) |
|||
.toOption(); |
|||
} |
|||
|
|||
private Try<BooleanExpression> getUsernamePredicate(final @Nonnull String username) { |
|||
|
|||
return Try |
|||
.of(() -> QTopicStarter.topicStarter) |
|||
.map(qTopicStarter -> qTopicStarter.starterUsername.like(username)); |
|||
} |
|||
|
|||
private Try<BooleanExpression> getUserIdPredicate(final @Nonnull Long userId) { |
|||
|
|||
return Try |
|||
.of(() -> QTopicStarter.topicStarter) |
|||
.map(qTopicStarter -> qTopicStarter.starterId.eq(userId)); |
|||
} |
|||
|
|||
private Try<BooleanExpression> getBoardTitlePredicate(final @Nonnull String boardTitle) { |
|||
|
|||
return Try |
|||
.of(() -> QTopicStarter.topicStarter) |
|||
.map(qTopicStarter -> qTopicStarter.boardTitle.like(boardTitle)); |
|||
} |
|||
|
|||
private Try<BooleanExpression> getBoardIdPredicate(final @Nonnull Long boardId) { |
|||
|
|||
return Try |
|||
.of(() -> QTopicStarter.topicStarter) |
|||
.map(qTopicStarter -> qTopicStarter.boardId.eq(boardId)); |
|||
} |
|||
|
|||
private Try<BooleanExpression> getTopicSubjectPredicate(final @Nonnull String topicSubject) { |
|||
|
|||
return Try |
|||
.of(() -> QTopicStarter.topicStarter) |
|||
.map(qTopicStarter -> qTopicStarter.topicSubject.like(topicSubject)); |
|||
} |
|||
|
|||
private Try<BooleanExpression> getTopicIdPredicate(final @Nonnull Long topicId) { |
|||
|
|||
return Try |
|||
.of(() -> QTopicStarter.topicStarter) |
|||
.map(qTopicStarter -> qTopicStarter.topicId.eq(topicId)); |
|||
} |
|||
} |
|||
@ -0,0 +1,8 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser; |
|||
|
|||
import io.vavr.control.Try; |
|||
|
|||
public interface TopicStarterParserService { |
|||
|
|||
Try<Void> parseTopicStarters(); |
|||
} |
|||
@ -0,0 +1,241 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser; |
|||
|
|||
import gr.thmmy.mthmmy.topicstarter.entity.topic.QTopicStarter; |
|||
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
|||
import gr.thmmy.mthmmy.topicstarter.repository.topic.TopicStarterRepository; |
|||
import gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util.TopicParserUtils; |
|||
import io.vavr.Tuple; |
|||
import io.vavr.collection.List; |
|||
import io.vavr.control.Option; |
|||
import io.vavr.control.Try; |
|||
import lombok.Data; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import okhttp3.*; |
|||
import org.jsoup.Jsoup; |
|||
import org.jsoup.nodes.Document; |
|||
import org.jsoup.nodes.Element; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.core.env.Environment; |
|||
import org.springframework.stereotype.Service; |
|||
|
|||
import java.util.stream.IntStream; |
|||
|
|||
import static gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util.BoardParserUtils.extractBoardIdFromUrl; |
|||
import static gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util.BoardParserUtils.parseSubBoards; |
|||
import static java.util.Objects.requireNonNull; |
|||
|
|||
@Service |
|||
@Data |
|||
@Slf4j |
|||
public class TopicStarterParserServiceImpl implements TopicStarterParserService { |
|||
/* Constants */ |
|||
private static final String baseUrl = "https://www.thmmy.gr/smf/index.php?action=forum"; |
|||
private static final String RECYCLING_BIN_BOARD_ID = "244.0"; |
|||
private static final String USERNAME_ENV_VAR = "TOPIC_STARTERS_USERNAME"; |
|||
private static final String PASSWORD_ENV_VAR = "TOPIC_STARTERS_PASSWORD"; |
|||
private static final HttpUrl loginUrl = HttpUrl.parse("https://www.thmmy.gr/smf/index.php?action=login2"); |
|||
|
|||
private final OkHttpClient client; |
|||
private final TopicStarterRepository topicStarterRepository; |
|||
|
|||
@Autowired |
|||
private Environment environment; |
|||
|
|||
@Override |
|||
public Try<Void> parseTopicStarters() { |
|||
|
|||
return login() |
|||
.flatMap(ignored -> parseBoard(baseUrl)); |
|||
} |
|||
|
|||
private Try<Void> parseBoard(final String url) { |
|||
requireNonNull(url, "url is null"); |
|||
|
|||
return Try.of(Request.Builder::new) |
|||
// Builds and executes request
|
|||
.map(requestBuilder -> requestBuilder |
|||
.url(url) |
|||
.build()) |
|||
.mapTry(request -> client |
|||
.newCall(request) |
|||
.execute()) |
|||
.flatMap(this::getResponseString) |
|||
.map(Jsoup::parse) |
|||
.flatMap(document -> parseSubBoards(document) // Parses the sub boards
|
|||
.flatMap(subBoards -> Option |
|||
.of(subBoards) |
|||
.toTry() |
|||
.flatMap(subBoardsNotNull -> Try |
|||
.run(() -> subBoardsNotNull |
|||
.stream() |
|||
.filter(subBoard -> !subBoard |
|||
.attr("href") |
|||
.contains("board=" + RECYCLING_BIN_BOARD_ID)) |
|||
.forEach(subBoard -> parseBoard(subBoard.attr("href"))) |
|||
) |
|||
) |
|||
) |
|||
// Parses the topics
|
|||
.flatMap(ignored -> parseTopics(document, url)) |
|||
); |
|||
} |
|||
|
|||
private Try<Void> parseTopics(final Document document, |
|||
final String boardUrl) { |
|||
requireNonNull(document, "document is null"); |
|||
requireNonNull(boardUrl, "boardUrl is null"); |
|||
|
|||
return Try.of(() -> document // Finds the number of pages in this board
|
|||
.select("a.navPages") |
|||
.last()) |
|||
.map(pageNumber -> Option |
|||
.of(pageNumber) |
|||
.map(Element::text) |
|||
.map(Integer::parseInt) |
|||
.getOrElse(1)) |
|||
.flatMap(numberOfPages -> Try // Parses the board title
|
|||
.of(() -> document |
|||
.select("div.nav>b>a") |
|||
.last() |
|||
.text()) |
|||
.flatMap(boardTitle -> extractBoardIdFromUrl(boardUrl) // Parses topics of the current page
|
|||
.flatMap(boardId -> saveTopics(document, boardTitle, boardUrl, boardId) |
|||
.flatMap(ignored -> IntStream // Parses the topics from the rest of the pages
|
|||
.range(1, numberOfPages) |
|||
.boxed() |
|||
.map(page -> Try // Builds the URL of the board for each page
|
|||
.of(() -> String.join(".", |
|||
boardUrl.substring(0, boardUrl.lastIndexOf(".")), |
|||
String.valueOf(page * 20)) |
|||
).flatMap(pageUrl -> Try |
|||
.of(Request.Builder::new) |
|||
.map(requestBuilder -> requestBuilder |
|||
.url(pageUrl) |
|||
.build() |
|||
) |
|||
) |
|||
.mapTry(request -> client |
|||
.newCall(request) |
|||
.execute()) |
|||
.flatMap(this::getResponseString) |
|||
.map(Jsoup::parse) |
|||
.flatMap(pageDocument -> saveTopics( |
|||
pageDocument, |
|||
boardTitle, |
|||
boardUrl, |
|||
boardId))) |
|||
.collect(List.collector()) |
|||
.transform(Try::sequence) |
|||
) |
|||
) |
|||
).map(ignored -> null)); |
|||
} |
|||
|
|||
private Try<Void> saveTopics(final Document document, |
|||
final String boardTitle, |
|||
final String boardUrl, |
|||
final Long boardId) { |
|||
requireNonNull(document, "document is null"); |
|||
requireNonNull(boardTitle, "boardTitle is null"); |
|||
requireNonNull(boardUrl, "boardUrl is null"); |
|||
requireNonNull(boardId, "boardId is null"); |
|||
|
|||
return Try // Finds this page's topics
|
|||
.of(() -> document |
|||
.select("table.bordercolor tbody>tr:not([class])")) |
|||
.flatMap(topics -> Try |
|||
.run(() -> topics |
|||
.forEach(topicRow -> TopicParserUtils |
|||
.parseTopic(topicRow, boardTitle, boardUrl, boardId) |
|||
.map(topicStarter -> savedTopicStarter(topicStarter) |
|||
.flatMap(savedTopicStarter -> updateSavedTopicStarter(savedTopicStarter, topicStarter)) |
|||
.getOrElse(topicStarter) |
|||
) |
|||
.map(topicStarterRepository::save) |
|||
) |
|||
) |
|||
); |
|||
} |
|||
|
|||
private Option<TopicStarter> savedTopicStarter(final TopicStarter topicStarter) { |
|||
requireNonNull(topicStarter, "topicStarter is null"); |
|||
|
|||
return Option |
|||
.of(QTopicStarter.topicStarter) |
|||
.map(qTopicStarter -> qTopicStarter.topicId.eq(topicStarter.getTopicId())) |
|||
.map(topicStarterRepository::findOne) |
|||
.flatMap(Option::ofOptional); |
|||
} |
|||
|
|||
private Option<TopicStarter> updateSavedTopicStarter(final TopicStarter savedTopicStarter, final TopicStarter newTopicStarter) { |
|||
requireNonNull(savedTopicStarter, "savedTopicStarter is null"); |
|||
requireNonNull(newTopicStarter, "newTopicStarter is null"); |
|||
|
|||
savedTopicStarter.setTopicId(newTopicStarter.getTopicId()); |
|||
savedTopicStarter.setTopicUrl(newTopicStarter.getTopicUrl()); |
|||
savedTopicStarter.setStarterUsername(newTopicStarter.getStarterUsername()); |
|||
savedTopicStarter.setStarterUrl(newTopicStarter.getStarterUrl()); |
|||
savedTopicStarter.setStarterId(newTopicStarter.getStarterId()); |
|||
savedTopicStarter.setBoardTitle(newTopicStarter.getBoardTitle()); |
|||
savedTopicStarter.setBoardUrl(newTopicStarter.getBoardUrl()); |
|||
savedTopicStarter.setBoardId(newTopicStarter.getBoardId()); |
|||
savedTopicStarter.setTopicSubject(newTopicStarter.getTopicSubject()); |
|||
savedTopicStarter.setNumberOfReplies(newTopicStarter.getNumberOfReplies()); |
|||
savedTopicStarter.setNumberOfViews(newTopicStarter.getNumberOfViews()); |
|||
|
|||
return Option.of(savedTopicStarter); |
|||
} |
|||
|
|||
private Try<Void> login() { |
|||
|
|||
return Option |
|||
.of(environment.getProperty(USERNAME_ENV_VAR)) |
|||
.map(username -> Option |
|||
.of(environment.getProperty(PASSWORD_ENV_VAR)) |
|||
.map(password -> Tuple.of(username, password)) |
|||
.getOrElseThrow(() -> new RuntimeException("Password is null")) |
|||
).map(loginSecrets -> Option |
|||
.of(loginUrl) |
|||
.map(loginUrlNotNull -> Try |
|||
.of(FormBody.Builder::new) |
|||
.map(builder -> builder |
|||
.add("user", loginSecrets._1) |
|||
.add("passwrd", loginSecrets._2) |
|||
.add("cookielength", "-1") // -1 is forever
|
|||
.build() |
|||
).flatMap(formBody -> Try |
|||
.of(Request.Builder::new) |
|||
.map(builder -> builder |
|||
.url(loginUrlNotNull) |
|||
.post(formBody) |
|||
.build() |
|||
) |
|||
).mapTry(request -> client |
|||
.newCall(request) |
|||
.execute() |
|||
).flatMap(response -> Try |
|||
.run(() -> response |
|||
.body() |
|||
.close() |
|||
) |
|||
) |
|||
).getOrElseThrow(() -> new RuntimeException("Login URL is null.")) |
|||
).getOrElseThrow(() -> new RuntimeException("Username is null")); |
|||
} |
|||
|
|||
private Try<String> getResponseString(final Response response) { |
|||
requireNonNull(response, "response is null"); |
|||
|
|||
// Checks response for null and closes response body
|
|||
return Try |
|||
.of(response::body) |
|||
.flatMap(responseBody -> Option |
|||
.of(responseBody) |
|||
.toTry() |
|||
.mapTry(ResponseBody::string) |
|||
.flatMap(responseBodyString -> Try |
|||
.run(responseBody::close) |
|||
.map(ignored -> responseBodyString)) |
|||
); |
|||
} |
|||
} |
|||
@ -0,0 +1,33 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util; |
|||
|
|||
import io.vavr.control.Try; |
|||
import org.jsoup.nodes.Document; |
|||
import org.jsoup.select.Elements; |
|||
|
|||
import java.util.regex.Pattern; |
|||
|
|||
import static java.util.Objects.requireNonNull; |
|||
|
|||
public abstract class BoardParserUtils { |
|||
|
|||
public static Try<Long> extractBoardIdFromUrl(final String url) { |
|||
requireNonNull(url, "url is null"); |
|||
|
|||
return Try.success(".+?board=([0-9]+)") |
|||
.map(regex -> Pattern.compile(regex, Pattern.MULTILINE)) |
|||
.map(pattern -> pattern.matcher(url)) |
|||
.map(matcher -> Try |
|||
.of(matcher::find) |
|||
.filter(aBoolean -> aBoolean) |
|||
.map(ignored -> Long.parseLong(matcher.group(1))) |
|||
.getOrElse(-1L) |
|||
); |
|||
} |
|||
|
|||
public static Try<Elements> parseSubBoards(final Document document) { |
|||
requireNonNull(document, "document is null"); |
|||
|
|||
return Try.of(() -> document |
|||
.select("div.tborder tbody tr.windowbg2 td>b>a[name^=b]")); |
|||
} |
|||
} |
|||
@ -0,0 +1,177 @@ |
|||
package gr.thmmy.mthmmy.topicstarter.service.topic.starter.parser.util; |
|||
|
|||
import gr.thmmy.mthmmy.topicstarter.entity.topic.TopicStarter; |
|||
import io.vavr.control.Option; |
|||
import io.vavr.control.Try; |
|||
import org.jsoup.nodes.Element; |
|||
import org.jsoup.select.Elements; |
|||
|
|||
import javax.annotation.Nullable; |
|||
import java.util.regex.Pattern; |
|||
|
|||
import static java.util.Objects.requireNonNull; |
|||
|
|||
public abstract class TopicParserUtils { |
|||
|
|||
public static Try<TopicStarter> parseTopic(final Element topicRow, |
|||
final String boardTitle, |
|||
final String boardUrl, |
|||
final Long boardId) { |
|||
requireNonNull(topicRow, "topicRow is null"); |
|||
requireNonNull(boardTitle, "boardTitle is null"); |
|||
requireNonNull(boardUrl, "boardUrl is null"); |
|||
requireNonNull(boardId, "boardId is null"); |
|||
|
|||
return Try.of(() -> topicRow |
|||
.select("td")) |
|||
.flatMap(topicColumns -> Try |
|||
.of(() -> topicColumns |
|||
.get(3) |
|||
.select("a") |
|||
.first()) |
|||
.flatMap(starterUrlElement -> Try |
|||
.of(TopicStarter::new) |
|||
.flatMap(topic -> parseTopicSubject(topicColumns) |
|||
.map(topicSubject -> { |
|||
topic.setTopicSubject(topicSubject); |
|||
|
|||
return topic; |
|||
}) |
|||
).flatMap(topic -> parseTopicUrl(topicColumns) |
|||
.flatMap(topicUrl -> extractTopicIdFromUrl(topicUrl) |
|||
.map(topicId -> { |
|||
topic.setTopicUrl(topicUrl); |
|||
topic.setTopicId(topicId); |
|||
|
|||
return topic; |
|||
}) |
|||
) |
|||
).flatMap(topic -> parseTopicStarterUsername(topicColumns) |
|||
.map(starterUsername -> { |
|||
topic.setStarterUsername(starterUsername); |
|||
|
|||
return topic; |
|||
}) |
|||
).flatMap(topic -> parseTopicStarterUrl(starterUrlElement) |
|||
.flatMap(topicStarterUrl -> extractTopicStarterIdFromUrl(topicStarterUrl) |
|||
.map(topicStarterId -> { |
|||
topic.setStarterUrl(topicStarterUrl); |
|||
topic.setStarterId(topicStarterId); |
|||
|
|||
return topic; |
|||
}) |
|||
) |
|||
).flatMap(topic -> parseTopicNumberOfReplies(topicColumns) |
|||
.map(numReplies -> { |
|||
topic.setNumberOfReplies(numReplies); |
|||
|
|||
return topic; |
|||
}) |
|||
).flatMap(topic -> parseTopicNumberOfViews(topicColumns) |
|||
.map(numViews -> { |
|||
topic.setNumberOfViews(numViews); |
|||
|
|||
return topic; |
|||
}) |
|||
).map(topic -> { |
|||
topic.setBoardTitle(boardTitle); |
|||
topic.setBoardId(boardId); |
|||
topic.setBoardUrl(boardUrl); |
|||
|
|||
return topic; |
|||
} |
|||
) |
|||
) |
|||
); |
|||
} |
|||
|
|||
private static Try<String> parseTopicSubject(final Elements topicColumns) { |
|||
requireNonNull(topicColumns, "topicColumns is null"); |
|||
|
|||
return Try |
|||
.of(() -> topicColumns |
|||
.get(2) |
|||
.select("span>a") |
|||
.first() |
|||
.text()); |
|||
} |
|||
|
|||
private static Try<String> parseTopicUrl(final Elements topicColumns) { |
|||
requireNonNull(topicColumns, "topicColumns is null"); |
|||
|
|||
return Try |
|||
.of(() -> topicColumns |
|||
.get(2) |
|||
.select("span>a") |
|||
.first() |
|||
.attr("href")); |
|||
} |
|||
|
|||
private static Try<String> parseTopicStarterUsername(final Elements topicColumns) { |
|||
requireNonNull(topicColumns, "topicColumns is null"); |
|||
|
|||
return Try |
|||
.of(() -> topicColumns |
|||
.get(3) |
|||
.text()); |
|||
} |
|||
|
|||
private static Try<String> parseTopicStarterUrl(final Element starterUrlEl) { |
|||
requireNonNull(starterUrlEl, "starterUrlEl is null"); |
|||
|
|||
return Try.of(() -> Option |
|||
.of(starterUrlEl) |
|||
.map(starterUrlElNotNull -> starterUrlElNotNull |
|||
.attr("href")) |
|||
.getOrElse(() -> null)); |
|||
} |
|||
|
|||
private static Try<Long> parseTopicNumberOfReplies(final Elements topicColumns) { |
|||
requireNonNull(topicColumns, "topicColumns is null"); |
|||
|
|||
return Try |
|||
.of(() -> topicColumns |
|||
.get(4) |
|||
.text()) |
|||
.map(Long::parseLong); |
|||
} |
|||
|
|||
private static Try<Long> parseTopicNumberOfViews(final Elements topicColumns) { |
|||
requireNonNull(topicColumns, "topicColumns is null"); |
|||
|
|||
return Try |
|||
.of(() -> topicColumns |
|||
.get(5) |
|||
.text()) |
|||
.map(Long::parseLong); |
|||
} |
|||
|
|||
private static Try<Long> extractTopicIdFromUrl(final String topicUrl) { |
|||
requireNonNull(topicUrl, "topicUrl is null"); |
|||
|
|||
return Try.success(".+?topic=([0-9]+)") |
|||
.map(regex -> Pattern.compile(regex, Pattern.MULTILINE)) |
|||
.map(pattern -> pattern.matcher(topicUrl)) |
|||
.map(matcher -> Try.of(matcher::find) |
|||
.filter(aBoolean -> aBoolean) |
|||
.map(ignored -> Long.parseLong(matcher.group(1))) |
|||
.getOrElse(-1L) |
|||
); |
|||
|
|||
} |
|||
|
|||
private static Try<Long> extractTopicStarterIdFromUrl(final @Nullable String topicStarterUrl) { |
|||
|
|||
return Option |
|||
.of(topicStarterUrl) |
|||
.map(topicStarterUrlNotNull -> Try |
|||
.of(() -> ".+?profile;u=([0-9]+)") |
|||
.map(regex -> Pattern.compile(regex, Pattern.MULTILINE)) |
|||
.map(pattern -> pattern.matcher(topicStarterUrlNotNull)) |
|||
.map(matcher -> Try.of(matcher::find) |
|||
.filter(aBoolean -> aBoolean) |
|||
.map(ignored -> Long.parseLong(matcher.group(1))) |
|||
.getOrElse(-1L)) |
|||
).getOrElse(Try.success(-1L)); |
|||
} |
|||
} |
|||
Loading…
Reference in new issue