From eeb5757a887a8101b54b8ed1c065d1938ff7367d Mon Sep 17 00:00:00 2001 From: Ezerous Date: Wed, 24 Oct 2018 13:22:56 +0300 Subject: [PATCH] Email deobfuscation optimizations --- .../create_content/NewTopicTask.java | 4 +- .../latestPosts/LatestPostsFragment.java | 6 ++- .../profile/stats/StatsFragment.java | 7 ++- .../profile/summary/SummaryFragment.java | 6 ++- .../mthmmy/activities/topic/Posting.java | 4 +- .../mthmmy/utils/parsing/ParseHelpers.java | 50 +++++++++++++------ 6 files changed, 53 insertions(+), 24 deletions(-) diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java b/app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java index 2cb321c1..985e0930 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java @@ -2,12 +2,12 @@ package gr.thmmy.mthmmy.activities.create_content; import android.os.AsyncTask; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import java.io.IOException; import gr.thmmy.mthmmy.base.BaseApplication; -import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import okhttp3.MultipartBody; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -44,7 +44,7 @@ public class NewTopicTask extends AsyncTask { String seqnum, sc, topic, createTopicUrl; try { Response response = client.newCall(request).execute(); - document = ParseHelpers.parse(response.body().string()); + document = Jsoup.parse(response.body().string()); seqnum = document.select("input[name=seqnum]").first().attr("value"); sc = document.select("input[name=sc]").first().attr("value"); diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java index a0d0c05a..b2fb112c 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java @@ -8,6 +8,7 @@ import android.view.ViewGroup; import android.widget.ProgressBar; import android.widget.Toast; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -29,6 +30,8 @@ import okhttp3.Request; import okhttp3.Response; import timber.log.Timber; +import static gr.thmmy.mthmmy.utils.parsing.ParseHelpers.deobfuscateElements; + /** * Use the {@link LatestPostsFragment#newInstance} factory method to create an instance of this fragment. */ @@ -163,7 +166,7 @@ public class LatestPostsFragment extends BaseFragment implements LatestPostsAdap .build(); try { Response response = BaseActivity.getClient().newCall(request).execute(); - return parseLatestPosts(ParseHelpers.parse(response.body().string())); + return parseLatestPosts(Jsoup.parse(response.body().string())); } catch (SSLHandshakeException e) { Timber.w("Certificate problem (please switch to unsafe connection)."); } catch (Exception e) { @@ -206,6 +209,7 @@ public class LatestPostsFragment extends BaseFragment implements LatestPostsAdap return true; } + deobfuscateElements(latestPostsRows, false); for (Element row : latestPostsRows) { String pTopicUrl, pTopicTitle, pDateTime, pPost; if (Integer.parseInt(row.attr("cellpadding")) == 4) { diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java index 4b3d661f..70e82fba 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java @@ -26,6 +26,7 @@ import com.github.mikephil.charting.data.LineDataSet; import com.github.mikephil.charting.formatter.IAxisValueFormatter; import com.github.mikephil.charting.formatter.PercentFormatter; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -39,12 +40,13 @@ import javax.net.ssl.SSLHandshakeException; import androidx.fragment.app.Fragment; import gr.thmmy.mthmmy.R; import gr.thmmy.mthmmy.base.BaseActivity; -import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import me.zhanghai.android.materialprogressbar.MaterialProgressBar; import okhttp3.Request; import okhttp3.Response; import timber.log.Timber; +import static gr.thmmy.mthmmy.utils.parsing.ParseHelpers.deobfuscateElements; + public class StatsFragment extends Fragment { /** * The key to use when putting profile's url String to {@link StatsFragment}'s Bundle. @@ -138,7 +140,7 @@ public class StatsFragment extends Fragment { .build(); try { Response response = BaseActivity.getClient().newCall(request).execute(); - return parseStats(ParseHelpers.parse(response.body().string())); + return parseStats(Jsoup.parse(response.body().string())); } catch (SSLHandshakeException e) { Timber.w("Certificate problem (please switch to unsafe connection)."); } catch (Exception e) { @@ -173,6 +175,7 @@ public class StatsFragment extends Fragment { return false; { Elements titleRows = statsPage.select("table.bordercolor[align]>tbody>tr.titlebg"); + deobfuscateElements(titleRows, false); generalStatisticsTitle = titleRows.first().text(); if (userHasPosts) { postingActivityByTimeTitle = titleRows.get(1).text(); diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java index cec78eda..264f7842 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java @@ -13,6 +13,7 @@ import android.webkit.WebView; import android.widget.LinearLayout; import android.widget.TextView; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -25,6 +26,8 @@ import gr.thmmy.mthmmy.R; import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import timber.log.Timber; +import static gr.thmmy.mthmmy.utils.parsing.ParseHelpers.deobfuscateElements; + /** * Use the {@link SummaryFragment#newInstance} factory method to create an instance of this fragment. @@ -68,7 +71,7 @@ public class SummaryFragment extends Fragment { @Override public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); - profileSummaryDocument = ParseHelpers.parse(getArguments().getString(PROFILE_DOCUMENT)); + profileSummaryDocument = Jsoup.parse(getArguments().getString(PROFILE_DOCUMENT)); parsedProfileSummaryData = new ArrayList<>(); } @@ -131,6 +134,7 @@ public class SummaryFragment extends Fragment { //Contains all summary's rows Elements summaryRows = profile.select(".bordercolor > tbody:nth-child(1) > tr:nth-child(2) tr"); + deobfuscateElements(summaryRows, false); for (Element summaryRow : summaryRows) { String rowText = summaryRow.text(), pHtml = ""; diff --git a/app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java b/app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java index 1a184d15..88c2f8a3 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java +++ b/app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java @@ -1,10 +1,10 @@ package gr.thmmy.mthmmy.activities.topic; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import java.io.IOException; -import gr.thmmy.mthmmy.utils.parsing.ParseHelpers; import okhttp3.Response; import timber.log.Timber; @@ -59,7 +59,7 @@ public class Posting { if (response.code() < 200 || response.code() >= 400) return REPLY_STATUS.OTHER_ERROR; String finalUrl = response.request().url().toString(); if (finalUrl.contains("action=post")) { - Document postErrorPage = ParseHelpers.parse(response.body().string()); + Document postErrorPage = Jsoup.parse(response.body().string()); String[] errors = postErrorPage.select("tr[id=errors] div[id=error_list]").first() .toString().split("
"); for (int i = 0; i < errors.length; ++i) { //TODO test diff --git a/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java b/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java index 2aecee04..022d9896 100644 --- a/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java +++ b/app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java @@ -10,6 +10,8 @@ import java.util.ArrayList; import java.util.regex.Matcher; import java.util.regex.Pattern; +import timber.log.Timber; + /** * This class consists exclusively of static classes (enums) and methods (excluding methods of inner * classes). It can be used to resolve a page's language and state or fix embedded videos html code @@ -190,7 +192,7 @@ public class ParseHelpers { } /** - * Method that adds email deobfuscation functionality to Jsoup.parse. + * Method that replaces CloudFlare-obfuscated emails with deobfuscated ones * Replace Jsoup.parse with this wherever needed * * @param html html to parse @@ -198,25 +200,41 @@ public class ParseHelpers { */ public static Document parse(String html){ Document document = Jsoup.parse(html); - Elements obfuscatedEmails = document.select("span.__cf_email__"); - for (Element obfuscatedEmail : obfuscatedEmails) { - String obfuscatedEmailStr = obfuscatedEmail.attr("data-cfemail"); - - //Deobfuscate - final StringBuilder stringBuilder = new StringBuilder(); - final int r = Integer.parseInt(obfuscatedEmailStr.substring(0, 2), 16); - for (int n = 2; n < obfuscatedEmailStr.length(); n += 2) { - final int i = Integer.parseInt(obfuscatedEmailStr.substring(n, n + 2), 16) ^ r; - stringBuilder.append(Character.toString((char) i)); - } + deobfuscateElements(document.select("span.__cf_email__"), true); + return document; + } - String deobfuscatedEmail = stringBuilder.toString(); + /** + * Use this method instead of parse() if you are targetting specific elements + */ + public static void deobfuscateElements(Elements elements, boolean found){ + if(!found) + elements = elements.select("span.__cf_email__"); - Element parent = obfuscatedEmail.parent(); + for (Element obfuscatedElement : elements) { + String deobfuscatedEmail = deobfuscateEmail(obfuscatedElement.attr("data-cfemail")); + Element parent = obfuscatedElement.parent(); if (parent.is("a")&&parent.attr("href").contains("email-protection")) parent.attr("href", "mailto:"+deobfuscatedEmail); - obfuscatedEmail.replaceWith(new TextNode(deobfuscatedEmail, "")); + obfuscatedElement.replaceWith(new TextNode(deobfuscatedEmail, "")); } - return document; + } + + + /** + * @param obfuscatedEmail CloudFlare-obfuscated email + * @return deobfuscated email + */ + private static String deobfuscateEmail(String obfuscatedEmail){ + //Deobfuscate + final StringBuilder stringBuilder = new StringBuilder(); + final int r = Integer.parseInt(obfuscatedEmail.substring(0, 2), 16); + for (int n = 2; n < obfuscatedEmail.length(); n += 2) { + final int i = Integer.parseInt(obfuscatedEmail.substring(n, n + 2), 16) ^ r; + stringBuilder.append(Character.toString((char) i)); + } + + Timber.i("Email deobfuscated."); + return stringBuilder.toString(); } }