From b4f830a5a5c22638658d2dbd50a7c8898bcc2862 Mon Sep 17 00:00:00 2001 From: Ezerous Date: Mon, 22 Oct 2018 20:32:59 +0300 Subject: [PATCH] Moved email deobfuscation in ParseHelpers --- .../create_content/NewTopicTask.java | 4 +- .../activities/profile/ProfileActivity.java | 3 +- .../latestPosts/LatestPostsFragment.java | 3 +- .../profile/stats/StatsFragment.java | 4 +- .../profile/summary/SummaryFragment.java | 3 +- .../mthmmy/activities/topic/Posting.java | 4 +- .../topic/tasks/PrepareForEditTask.java | 4 +- .../topic/tasks/PrepareForReply.java | 4 +- .../activities/topic/tasks/TopicTask.java | 3 +- .../gr/thmmy/mthmmy/base/BaseApplication.java | 12 +---- .../thmmy/mthmmy/session/SessionManager.java | 6 +-- .../thmmy/mthmmy/utils/EmailDeobfuscator.java | 44 ------------------- .../gr/thmmy/mthmmy/utils/NetworkTask.java | 4 +- .../mthmmy/utils/parsing/ParseHelpers.java | 36 ++++++++++++++- .../thmmy/mthmmy/utils/parsing/ParseTask.java | 3 +- 15 files changed, 57 insertions(+), 80 deletions(-) delete mode 100644 app/src/main/java/gr/thmmy/mthmmy/utils/EmailDeobfuscator.java diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java b/app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java index 985e0930..2cb321c1 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java @@ -2,12 +2,12 @@ package gr.thmmy.mthmmy.activities.create_content; import android.os.AsyncTask; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import java.io.IOException; import gr.thmmy.mthmmy.base.BaseApplication; +import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import okhttp3.MultipartBody; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -44,7 +44,7 @@ public class NewTopicTask extends AsyncTask { String seqnum, sc, topic, createTopicUrl; try { Response response = client.newCall(request).execute(); - document = Jsoup.parse(response.body().string()); + document = ParseHelpers.parse(response.body().string()); seqnum = document.select("input[name=seqnum]").first().attr("value"); sc = document.select("input[name=sc]").first().attr("value"); diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/ProfileActivity.java b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/ProfileActivity.java index 99015a2b..70f95ac4 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/ProfileActivity.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/ProfileActivity.java @@ -49,6 +49,7 @@ import gr.thmmy.mthmmy.model.PostSummary; import gr.thmmy.mthmmy.model.ThmmyPage; import gr.thmmy.mthmmy.utils.CenterVerticalSpan; import gr.thmmy.mthmmy.utils.CircleTransform; +import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import me.zhanghai.android.materialprogressbar.MaterialProgressBar; import okhttp3.Request; import okhttp3.Response; @@ -241,7 +242,7 @@ public class ProfileActivity extends BaseActivity implements LatestPostsFragment .build(); try { Response response = client.newCall(request).execute(); - profilePage = Jsoup.parse(response.body().string()); + profilePage = ParseHelpers.parse(response.body().string()); Elements contentsTable = profilePage. select(".bordercolor > tbody:nth-child(1) > tr:nth-child(2) tbody"); diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java index 9736d3fd..a0d0c05a 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java @@ -8,7 +8,6 @@ import android.view.ViewGroup; import android.widget.ProgressBar; import android.widget.Toast; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -164,7 +163,7 @@ public class LatestPostsFragment extends BaseFragment implements LatestPostsAdap .build(); try { Response response = BaseActivity.getClient().newCall(request).execute(); - return parseLatestPosts(Jsoup.parse(response.body().string())); + return parseLatestPosts(ParseHelpers.parse(response.body().string())); } catch (SSLHandshakeException e) { Timber.w("Certificate problem (please switch to unsafe connection)."); } catch (Exception e) { diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java index eca17c0e..4b3d661f 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java @@ -26,7 +26,6 @@ import com.github.mikephil.charting.data.LineDataSet; import com.github.mikephil.charting.formatter.IAxisValueFormatter; import com.github.mikephil.charting.formatter.PercentFormatter; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -40,6 +39,7 @@ import javax.net.ssl.SSLHandshakeException; import androidx.fragment.app.Fragment; import gr.thmmy.mthmmy.R; import gr.thmmy.mthmmy.base.BaseActivity; +import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import me.zhanghai.android.materialprogressbar.MaterialProgressBar; import okhttp3.Request; import okhttp3.Response; @@ -138,7 +138,7 @@ public class StatsFragment extends Fragment { .build(); try { Response response = BaseActivity.getClient().newCall(request).execute(); - return parseStats(Jsoup.parse(response.body().string())); + return parseStats(ParseHelpers.parse(response.body().string())); } catch (SSLHandshakeException e) { Timber.w("Certificate problem (please switch to unsafe connection)."); } catch (Exception e) { diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java index 867707ee..cec78eda 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java @@ -13,7 +13,6 @@ import android.webkit.WebView; import android.widget.LinearLayout; import android.widget.TextView; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -69,7 +68,7 @@ public class SummaryFragment extends Fragment { @Override public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); - profileSummaryDocument = Jsoup.parse(getArguments().getString(PROFILE_DOCUMENT)); + profileSummaryDocument = ParseHelpers.parse(getArguments().getString(PROFILE_DOCUMENT)); parsedProfileSummaryData = new ArrayList<>(); } diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java b/app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java index 88c2f8a3..1a184d15 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java @@ -1,10 +1,10 @@ package gr.thmmy.mthmmy.activities.topic; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import java.io.IOException; +import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import okhttp3.Response; import timber.log.Timber; @@ -59,7 +59,7 @@ public class Posting { if (response.code() < 200 || response.code() >= 400) return REPLY_STATUS.OTHER_ERROR; String finalUrl = response.request().url().toString(); if (finalUrl.contains("action=post")) { - Document postErrorPage = Jsoup.parse(response.body().string()); + Document postErrorPage = ParseHelpers.parse(response.body().string()); String[] errors = postErrorPage.select("tr[id=errors] div[id=error_list]").first() .toString().split("
"); for (int i = 0; i < errors.length; ++i) { //TODO test diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/PrepareForEditTask.java b/app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/PrepareForEditTask.java index 5864e635..6efc2c03 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/PrepareForEditTask.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/PrepareForEditTask.java @@ -2,7 +2,6 @@ package gr.thmmy.mthmmy.activities.topic.tasks; import android.os.AsyncTask; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Selector; @@ -10,6 +9,7 @@ import org.jsoup.select.Selector; import java.io.IOException; import gr.thmmy.mthmmy.base.BaseApplication; +import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -45,7 +45,7 @@ public class PrepareForEditTask extends AsyncTask { .build(); try { Response response = BaseApplication.getInstance().getClient().newCall(request).execute(); - topic = Jsoup.parse(response.body().string()); + topic = ParseHelpers.parse(response.body().string()); ParseHelpers.Language language = ParseHelpers.Language.getLanguage(topic); diff --git a/app/src/main/java/gr/thmmy/mthmmy/base/BaseApplication.java b/app/src/main/java/gr/thmmy/mthmmy/base/BaseApplication.java index e218c9f4..0c0eda0e 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/base/BaseApplication.java +++ b/app/src/main/java/gr/thmmy/mthmmy/base/BaseApplication.java @@ -38,11 +38,8 @@ import io.fabric.sdk.android.Fabric; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Request; -import okhttp3.Response; import timber.log.Timber; -import static gr.thmmy.mthmmy.utils.EmailDeobfuscator.deobfuscate; - public class BaseApplication extends Application { private static BaseApplication baseApplication; //BaseApplication singleton @@ -103,14 +100,7 @@ public class BaseApplication extends Application { request = request.newBuilder().url(newUrl).build(); } } - - Response response = chain.proceed(request); - try { - response = deobfuscate(response); - } catch (Exception e) { - Timber.e(e, "Email deobfuscation error."); - } - return response; + return chain.proceed(request); }) .connectTimeout(30, TimeUnit.SECONDS) .writeTimeout(30, TimeUnit.SECONDS) diff --git a/app/src/main/java/gr/thmmy/mthmmy/session/SessionManager.java b/app/src/main/java/gr/thmmy/mthmmy/session/SessionManager.java index 97e1dddb..4121c1de 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/session/SessionManager.java +++ b/app/src/main/java/gr/thmmy/mthmmy/session/SessionManager.java @@ -5,7 +5,6 @@ import android.content.SharedPreferences; import com.franmontiel.persistentcookiejar.PersistentCookieJar; import com.franmontiel.persistentcookiejar.persistence.SharedPrefsCookiePersistor; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; @@ -18,6 +17,7 @@ import java.util.regex.Pattern; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import gr.thmmy.mthmmy.utils.parsing.ParseException; +import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import okhttp3.Cookie; import okhttp3.FormBody; import okhttp3.HttpUrl; @@ -109,7 +109,7 @@ public class SessionManager { try { //Make request & handle response Response response = client.newCall(request).execute(); - Document document = Jsoup.parse(response.body().string()); + Document document = ParseHelpers.parse(response.body().string()); if (validateRetrievedCookies()) { @@ -215,7 +215,7 @@ public class SessionManager { try { //Make request & handle response Response response = client.newCall(request).execute(); - Document document = Jsoup.parse(response.body().string()); + Document document = ParseHelpers.parse(response.body().string()); Elements loginButton = document.select("[value=Login]"); //Attempt to find login button if (!loginButton.isEmpty()) //If login button exists, logout was successful diff --git a/app/src/main/java/gr/thmmy/mthmmy/utils/EmailDeobfuscator.java b/app/src/main/java/gr/thmmy/mthmmy/utils/EmailDeobfuscator.java deleted file mode 100644 index 56ee76ad..00000000 --- a/app/src/main/java/gr/thmmy/mthmmy/utils/EmailDeobfuscator.java +++ /dev/null @@ -1,44 +0,0 @@ -package gr.thmmy.mthmmy.utils; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.nodes.TextNode; -import org.jsoup.select.Elements; - -import java.io.IOException; - -import okhttp3.MediaType; -import okhttp3.Response; -import okhttp3.ResponseBody; - -//Deobfuscates Cloudflare-obfuscated emails -public class EmailDeobfuscator { - public static Response deobfuscate(Response response) throws IOException { - String responseBody = response.body().string(); - Document document = Jsoup.parse(responseBody); - Elements obfuscatedEmails = document.select("span.__cf_email__"); - for (Element obfuscatedEmail : obfuscatedEmails) { - String email = deobfuscateEmail(obfuscatedEmail.attr("data-cfemail")); - Element parent = obfuscatedEmail.parent(); - if (parent.is("a")&&parent.attr("href").contains("email-protection")) - parent.attr("href", "mailto:"+email); - obfuscatedEmail.replaceWith(new TextNode(email, "")); - } - - MediaType contentType = response.body().contentType(); - ResponseBody body = ResponseBody.create(contentType, document.toString()); - return response.newBuilder().body(body).build(); - } - - - private static String deobfuscateEmail(final String encodedString) { - final StringBuilder email = new StringBuilder(); - final int r = Integer.parseInt(encodedString.substring(0, 2), 16); - for (int n = 2; n < encodedString.length(); n += 2) { - final int i = Integer.parseInt(encodedString.substring(n, n+2), 16) ^ r; - email.append(Character.toString ((char) i)); - } - return email.toString(); - } -} diff --git a/app/src/main/java/gr/thmmy/mthmmy/utils/NetworkTask.java b/app/src/main/java/gr/thmmy/mthmmy/utils/NetworkTask.java index cb613f26..c61dbb72 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/utils/NetworkTask.java +++ b/app/src/main/java/gr/thmmy/mthmmy/utils/NetworkTask.java @@ -1,12 +1,12 @@ package gr.thmmy.mthmmy.utils; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import java.io.IOException; import gr.thmmy.mthmmy.base.BaseApplication; import gr.thmmy.mthmmy.utils.parsing.ParseException; +import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -49,7 +49,7 @@ public abstract class NetworkTask extends ExternalAsyncTask return new Parcel<>(NetworkResultCodes.NETWORK_ERROR, null); } try { - T data = performTask(Jsoup.parse(responseBodyString), response); + T data = performTask(ParseHelpers.parse(responseBodyString), response); int resultCode = getResultCode(response, data); return new Parcel<>(resultCode, data); } catch (ParseException pe) { diff --git a/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java b/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java index 2208ff04..2aecee04 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java +++ b/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java @@ -1,7 +1,9 @@ package gr.thmmy.mthmmy.utils.parsing; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.nodes.TextNode; import org.jsoup.select.Elements; import java.util.ArrayList; @@ -10,7 +12,8 @@ import java.util.regex.Pattern; /** * This class consists exclusively of static classes (enums) and methods (excluding methods of inner - * classes). It can be used to resolve a page's language and state or fix embedded videos html code. + * classes). It can be used to resolve a page's language and state or fix embedded videos html code + * and obfuscated emails. */ public class ParseHelpers { /** @@ -185,4 +188,35 @@ public class ParseHelpers { return forumUrl + topicURL.substring(baseUrlMatcher.start(), baseUrlMatcher.end()); else return ""; } + + /** + * Method that adds email deobfuscation functionality to Jsoup.parse. + * Replace Jsoup.parse with this wherever needed + * + * @param html html to parse + * @return a document with deobfuscated emails + */ + public static Document parse(String html){ + Document document = Jsoup.parse(html); + Elements obfuscatedEmails = document.select("span.__cf_email__"); + for (Element obfuscatedEmail : obfuscatedEmails) { + String obfuscatedEmailStr = obfuscatedEmail.attr("data-cfemail"); + + //Deobfuscate + final StringBuilder stringBuilder = new StringBuilder(); + final int r = Integer.parseInt(obfuscatedEmailStr.substring(0, 2), 16); + for (int n = 2; n < obfuscatedEmailStr.length(); n += 2) { + final int i = Integer.parseInt(obfuscatedEmailStr.substring(n, n + 2), 16) ^ r; + stringBuilder.append(Character.toString((char) i)); + } + + String deobfuscatedEmail = stringBuilder.toString(); + + Element parent = obfuscatedEmail.parent(); + if (parent.is("a")&&parent.attr("href").contains("email-protection")) + parent.attr("href", "mailto:"+deobfuscatedEmail); + obfuscatedEmail.replaceWith(new TextNode(deobfuscatedEmail, "")); + } + return document; + } } diff --git a/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseTask.java b/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseTask.java index 4f335025..6936100f 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseTask.java +++ b/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseTask.java @@ -3,7 +3,6 @@ package gr.thmmy.mthmmy.utils.parsing; import android.os.AsyncTask; import android.widget.Toast; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import java.io.IOException; @@ -42,7 +41,7 @@ public abstract class ParseTask extends AsyncTask