Browse Source

Moved email deobfuscation in ParseHelpers

pull/61/merge
Ezerous 6 years ago
parent
commit
b4f830a5a5
No known key found for this signature in database GPG Key ID: 262B2954BBA319E3
  1. 4
      app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java
  2. 3
      app/src/main/java/gr/thmmy/mthmmy/activities/profile/ProfileActivity.java
  3. 3
      app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java
  4. 4
      app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java
  5. 3
      app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java
  6. 4
      app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java
  7. 4
      app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/PrepareForEditTask.java
  8. 4
      app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/PrepareForReply.java
  9. 3
      app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/TopicTask.java
  10. 12
      app/src/main/java/gr/thmmy/mthmmy/base/BaseApplication.java
  11. 6
      app/src/main/java/gr/thmmy/mthmmy/session/SessionManager.java
  12. 44
      app/src/main/java/gr/thmmy/mthmmy/utils/EmailDeobfuscator.java
  13. 4
      app/src/main/java/gr/thmmy/mthmmy/utils/NetworkTask.java
  14. 36
      app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java
  15. 3
      app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseTask.java

4
app/src/main/java/gr/thmmy/mthmmy/activities/create_content/NewTopicTask.java

@ -2,12 +2,12 @@ package gr.thmmy.mthmmy.activities.create_content;
import android.os.AsyncTask;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException;
import gr.thmmy.mthmmy.base.BaseApplication;
import gr.thmmy.mthmmy.utils.parsing.ParseHelpers;
import okhttp3.MultipartBody;
import okhttp3.OkHttpClient;
import okhttp3.Request;
@ -44,7 +44,7 @@ public class NewTopicTask extends AsyncTask<String, Void, Boolean> {
String seqnum, sc, topic, createTopicUrl;
try {
Response response = client.newCall(request).execute();
document = Jsoup.parse(response.body().string());
document = ParseHelpers.parse(response.body().string());
seqnum = document.select("input[name=seqnum]").first().attr("value");
sc = document.select("input[name=sc]").first().attr("value");

3
app/src/main/java/gr/thmmy/mthmmy/activities/profile/ProfileActivity.java

@ -49,6 +49,7 @@ import gr.thmmy.mthmmy.model.PostSummary;
import gr.thmmy.mthmmy.model.ThmmyPage;
import gr.thmmy.mthmmy.utils.CenterVerticalSpan;
import gr.thmmy.mthmmy.utils.CircleTransform;
import gr.thmmy.mthmmy.utils.parsing.ParseHelpers;
import me.zhanghai.android.materialprogressbar.MaterialProgressBar;
import okhttp3.Request;
import okhttp3.Response;
@ -241,7 +242,7 @@ public class ProfileActivity extends BaseActivity implements LatestPostsFragment
.build();
try {
Response response = client.newCall(request).execute();
profilePage = Jsoup.parse(response.body().string());
profilePage = ParseHelpers.parse(response.body().string());
Elements contentsTable = profilePage.
select(".bordercolor > tbody:nth-child(1) > tr:nth-child(2) tbody");

3
app/src/main/java/gr/thmmy/mthmmy/activities/profile/latestPosts/LatestPostsFragment.java

@ -8,7 +8,6 @@ import android.view.ViewGroup;
import android.widget.ProgressBar;
import android.widget.Toast;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
@ -164,7 +163,7 @@ public class LatestPostsFragment extends BaseFragment implements LatestPostsAdap
.build();
try {
Response response = BaseActivity.getClient().newCall(request).execute();
return parseLatestPosts(Jsoup.parse(response.body().string()));
return parseLatestPosts(ParseHelpers.parse(response.body().string()));
} catch (SSLHandshakeException e) {
Timber.w("Certificate problem (please switch to unsafe connection).");
} catch (Exception e) {

4
app/src/main/java/gr/thmmy/mthmmy/activities/profile/stats/StatsFragment.java

@ -26,7 +26,6 @@ import com.github.mikephil.charting.data.LineDataSet;
import com.github.mikephil.charting.formatter.IAxisValueFormatter;
import com.github.mikephil.charting.formatter.PercentFormatter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
@ -40,6 +39,7 @@ import javax.net.ssl.SSLHandshakeException;
import androidx.fragment.app.Fragment;
import gr.thmmy.mthmmy.R;
import gr.thmmy.mthmmy.base.BaseActivity;
import gr.thmmy.mthmmy.utils.parsing.ParseHelpers;
import me.zhanghai.android.materialprogressbar.MaterialProgressBar;
import okhttp3.Request;
import okhttp3.Response;
@ -138,7 +138,7 @@ public class StatsFragment extends Fragment {
.build();
try {
Response response = BaseActivity.getClient().newCall(request).execute();
return parseStats(Jsoup.parse(response.body().string()));
return parseStats(ParseHelpers.parse(response.body().string()));
} catch (SSLHandshakeException e) {
Timber.w("Certificate problem (please switch to unsafe connection).");
} catch (Exception e) {

3
app/src/main/java/gr/thmmy/mthmmy/activities/profile/summary/SummaryFragment.java

@ -13,7 +13,6 @@ import android.webkit.WebView;
import android.widget.LinearLayout;
import android.widget.TextView;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
@ -69,7 +68,7 @@ public class SummaryFragment extends Fragment {
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
profileSummaryDocument = Jsoup.parse(getArguments().getString(PROFILE_DOCUMENT));
profileSummaryDocument = ParseHelpers.parse(getArguments().getString(PROFILE_DOCUMENT));
parsedProfileSummaryData = new ArrayList<>();
}

4
app/src/main/java/gr/thmmy/mthmmy/activities/topic/Posting.java

@ -1,10 +1,10 @@
package gr.thmmy.mthmmy.activities.topic;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException;
import gr.thmmy.mthmmy.utils.parsing.ParseHelpers;
import okhttp3.Response;
import timber.log.Timber;
@ -59,7 +59,7 @@ public class Posting {
if (response.code() < 200 || response.code() >= 400) return REPLY_STATUS.OTHER_ERROR;
String finalUrl = response.request().url().toString();
if (finalUrl.contains("action=post")) {
Document postErrorPage = Jsoup.parse(response.body().string());
Document postErrorPage = ParseHelpers.parse(response.body().string());
String[] errors = postErrorPage.select("tr[id=errors] div[id=error_list]").first()
.toString().split("<br>");
for (int i = 0; i < errors.length; ++i) { //TODO test

4
app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/PrepareForEditTask.java

@ -2,7 +2,6 @@ package gr.thmmy.mthmmy.activities.topic.tasks;
import android.os.AsyncTask;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Selector;
@ -10,6 +9,7 @@ import org.jsoup.select.Selector;
import java.io.IOException;
import gr.thmmy.mthmmy.base.BaseApplication;
import gr.thmmy.mthmmy.utils.parsing.ParseHelpers;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
@ -45,7 +45,7 @@ public class PrepareForEditTask extends AsyncTask<String, Void, PrepareForEditRe
String postText, commitEditURL, numReplies, seqnum, sc, topic;
OkHttpClient client = BaseApplication.getInstance().getClient();
Response response = client.newCall(request).execute();
document = Jsoup.parse(response.body().string());
document = ParseHelpers.parse(response.body().string());
Element message = document.select("textarea").first();
postText = message.text();

4
app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/PrepareForReply.java

@ -2,13 +2,13 @@ package gr.thmmy.mthmmy.activities.topic.tasks;
import android.os.AsyncTask;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Selector;
import java.io.IOException;
import gr.thmmy.mthmmy.base.BaseApplication;
import gr.thmmy.mthmmy.utils.parsing.ParseHelpers;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
@ -42,7 +42,7 @@ public class PrepareForReply extends AsyncTask<Integer, Void, PrepareForReplyRes
String numReplies, seqnum, sc, topic;
try {
Response response = client.newCall(request).execute();
document = Jsoup.parse(response.body().string());
document = ParseHelpers.parse(response.body().string());
numReplies = replyPageUrl.substring(replyPageUrl.indexOf("num_replies=") + 12);
seqnum = document.select("input[name=seqnum]").first().attr("value");

3
app/src/main/java/gr/thmmy/mthmmy/activities/topic/tasks/TopicTask.java

@ -2,7 +2,6 @@ package gr.thmmy.mthmmy.activities.topic.tasks;
import android.os.AsyncTask;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -61,7 +60,7 @@ public class TopicTask extends AsyncTask<String, Void, TopicTaskResult> {
.build();
try {
Response response = BaseApplication.getInstance().getClient().newCall(request).execute();
topic = Jsoup.parse(response.body().string());
topic = ParseHelpers.parse(response.body().string());
ParseHelpers.Language language = ParseHelpers.Language.getLanguage(topic);

12
app/src/main/java/gr/thmmy/mthmmy/base/BaseApplication.java

@ -38,11 +38,8 @@ import io.fabric.sdk.android.Fabric;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import timber.log.Timber;
import static gr.thmmy.mthmmy.utils.EmailDeobfuscator.deobfuscate;
public class BaseApplication extends Application {
private static BaseApplication baseApplication; //BaseApplication singleton
@ -103,14 +100,7 @@ public class BaseApplication extends Application {
request = request.newBuilder().url(newUrl).build();
}
}
Response response = chain.proceed(request);
try {
response = deobfuscate(response);
} catch (Exception e) {
Timber.e(e, "Email deobfuscation error.");
}
return response;
return chain.proceed(request);
})
.connectTimeout(30, TimeUnit.SECONDS)
.writeTimeout(30, TimeUnit.SECONDS)

6
app/src/main/java/gr/thmmy/mthmmy/session/SessionManager.java

@ -5,7 +5,6 @@ import android.content.SharedPreferences;
import com.franmontiel.persistentcookiejar.PersistentCookieJar;
import com.franmontiel.persistentcookiejar.persistence.SharedPrefsCookiePersistor;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
@ -18,6 +17,7 @@ import java.util.regex.Pattern;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import gr.thmmy.mthmmy.utils.parsing.ParseException;
import gr.thmmy.mthmmy.utils.parsing.ParseHelpers;
import okhttp3.Cookie;
import okhttp3.FormBody;
import okhttp3.HttpUrl;
@ -109,7 +109,7 @@ public class SessionManager {
try {
//Make request & handle response
Response response = client.newCall(request).execute();
Document document = Jsoup.parse(response.body().string());
Document document = ParseHelpers.parse(response.body().string());
if (validateRetrievedCookies())
{
@ -215,7 +215,7 @@ public class SessionManager {
try {
//Make request & handle response
Response response = client.newCall(request).execute();
Document document = Jsoup.parse(response.body().string());
Document document = ParseHelpers.parse(response.body().string());
Elements loginButton = document.select("[value=Login]"); //Attempt to find login button
if (!loginButton.isEmpty()) //If login button exists, logout was successful

44
app/src/main/java/gr/thmmy/mthmmy/utils/EmailDeobfuscator.java

@ -1,44 +0,0 @@
package gr.thmmy.mthmmy.utils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;
import java.io.IOException;
import okhttp3.MediaType;
import okhttp3.Response;
import okhttp3.ResponseBody;
//Deobfuscates Cloudflare-obfuscated emails
public class EmailDeobfuscator {
public static Response deobfuscate(Response response) throws IOException {
String responseBody = response.body().string();
Document document = Jsoup.parse(responseBody);
Elements obfuscatedEmails = document.select("span.__cf_email__");
for (Element obfuscatedEmail : obfuscatedEmails) {
String email = deobfuscateEmail(obfuscatedEmail.attr("data-cfemail"));
Element parent = obfuscatedEmail.parent();
if (parent.is("a")&&parent.attr("href").contains("email-protection"))
parent.attr("href", "mailto:"+email);
obfuscatedEmail.replaceWith(new TextNode(email, ""));
}
MediaType contentType = response.body().contentType();
ResponseBody body = ResponseBody.create(contentType, document.toString());
return response.newBuilder().body(body).build();
}
private static String deobfuscateEmail(final String encodedString) {
final StringBuilder email = new StringBuilder();
final int r = Integer.parseInt(encodedString.substring(0, 2), 16);
for (int n = 2; n < encodedString.length(); n += 2) {
final int i = Integer.parseInt(encodedString.substring(n, n+2), 16) ^ r;
email.append(Character.toString ((char) i));
}
return email.toString();
}
}

4
app/src/main/java/gr/thmmy/mthmmy/utils/NetworkTask.java

@ -1,12 +1,12 @@
package gr.thmmy.mthmmy.utils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException;
import gr.thmmy.mthmmy.base.BaseApplication;
import gr.thmmy.mthmmy.utils.parsing.ParseException;
import gr.thmmy.mthmmy.utils.parsing.ParseHelpers;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
@ -49,7 +49,7 @@ public abstract class NetworkTask<T> extends ExternalAsyncTask<String, Parcel<T>
return new Parcel<>(NetworkResultCodes.NETWORK_ERROR, null);
}
try {
T data = performTask(Jsoup.parse(responseBodyString), response);
T data = performTask(ParseHelpers.parse(responseBodyString), response);
int resultCode = getResultCode(response, data);
return new Parcel<>(resultCode, data);
} catch (ParseException pe) {

36
app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseHelpers.java

@ -1,7 +1,9 @@
package gr.thmmy.mthmmy.utils.parsing;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;
import java.util.ArrayList;
@ -10,7 +12,8 @@ import java.util.regex.Pattern;
/**
* This class consists exclusively of static classes (enums) and methods (excluding methods of inner
* classes). It can be used to resolve a page's language and state or fix embedded videos html code.
* classes). It can be used to resolve a page's language and state or fix embedded videos html code
* and obfuscated emails.
*/
public class ParseHelpers {
/**
@ -185,4 +188,35 @@ public class ParseHelpers {
return forumUrl + topicURL.substring(baseUrlMatcher.start(), baseUrlMatcher.end());
else return "";
}
/**
* Method that adds email deobfuscation functionality to Jsoup.parse.
* Replace Jsoup.parse with this wherever needed
*
* @param html html to parse
* @return a document with deobfuscated emails
*/
public static Document parse(String html){
Document document = Jsoup.parse(html);
Elements obfuscatedEmails = document.select("span.__cf_email__");
for (Element obfuscatedEmail : obfuscatedEmails) {
String obfuscatedEmailStr = obfuscatedEmail.attr("data-cfemail");
//Deobfuscate
final StringBuilder stringBuilder = new StringBuilder();
final int r = Integer.parseInt(obfuscatedEmailStr.substring(0, 2), 16);
for (int n = 2; n < obfuscatedEmailStr.length(); n += 2) {
final int i = Integer.parseInt(obfuscatedEmailStr.substring(n, n + 2), 16) ^ r;
stringBuilder.append(Character.toString((char) i));
}
String deobfuscatedEmail = stringBuilder.toString();
Element parent = obfuscatedEmail.parent();
if (parent.is("a")&&parent.attr("href").contains("email-protection"))
parent.attr("href", "mailto:"+deobfuscatedEmail);
obfuscatedEmail.replaceWith(new TextNode(deobfuscatedEmail, ""));
}
return document;
}
}

3
app/src/main/java/gr/thmmy/mthmmy/utils/parsing/ParseTask.java

@ -3,7 +3,6 @@ package gr.thmmy.mthmmy.utils.parsing;
import android.os.AsyncTask;
import android.widget.Toast;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException;
@ -42,7 +41,7 @@ public abstract class ParseTask extends AsyncTask<String, Void, ParseTask.Result
Request request = prepareRequest(params);
try {
Response response = BaseApplication.getInstance().getClient().newCall(request).execute();
Document document = Jsoup.parse(response.body().string());
Document document = ParseHelpers.parse(response.body().string());
parse(document);
postParsing();
return ResultCode.SUCCESS;

Loading…
Cancel
Save