feat: implement background fetch worker (cont.)

app/proguard-rules.pro

@@ -23,6 +23,9 @@
 -dontwarn kotlin.reflect.jvm.internal.**
 
 -keep class kotlin.reflect.jvm.internal.** { *; }
+-keep class it.skrape.fetcher.** { *; }
+-keep class io.ktor.client.** { *; }
+-keep,allowobfuscation,allowshrinking class kotlin.coroutines.Continuation
 
 -keep interface javax.annotation.Nullable
 
@@ -119,4 +122,84 @@
 -dontwarn org.ietf.jgss.GSSManager
 -dontwarn org.ietf.jgss.GSSName
 -dontwarn org.ietf.jgss.Oid
--dontwarn sun.reflect.Reflection
+-dontwarn sun.reflect.Reflection
+# Please add these rules to your existing keep rules in order to suppress warnings.
+# This is generated automatically by the Android Gradle plugin.
+-dontwarn edu.umd.cs.findbugs.annotations.SuppressFBWarnings
+-dontwarn java.awt.AWTEvent
+-dontwarn java.awt.CardLayout
+-dontwarn java.awt.Color
+-dontwarn java.awt.Component
+-dontwarn java.awt.Container
+-dontwarn java.awt.Dialog
+-dontwarn java.awt.Dimension
+-dontwarn java.awt.FlowLayout
+-dontwarn java.awt.Frame
+-dontwarn java.awt.Graphics
+-dontwarn java.awt.GridBagConstraints
+-dontwarn java.awt.GridBagLayout
+-dontwarn java.awt.GridLayout
+-dontwarn java.awt.Insets
+-dontwarn java.awt.Label
+-dontwarn java.awt.LayoutManager
+-dontwarn java.awt.SystemColor
+-dontwarn java.awt.TextArea
+-dontwarn java.awt.TextComponent
+-dontwarn java.awt.TextField
+-dontwarn java.awt.Toolkit
+-dontwarn java.awt.Window
+-dontwarn java.awt.event.ActionEvent
+-dontwarn java.awt.event.ActionListener
+-dontwarn java.awt.event.TextEvent
+-dontwarn java.awt.event.TextListener
+-dontwarn java.awt.event.WindowAdapter
+-dontwarn java.awt.event.WindowEvent
+-dontwarn java.awt.event.WindowListener
+-dontwarn javax.mail.Address
+-dontwarn javax.mail.Authenticator
+-dontwarn javax.mail.BodyPart
+-dontwarn javax.mail.Message$RecipientType
+-dontwarn javax.mail.Message
+-dontwarn javax.mail.Multipart
+-dontwarn javax.mail.Session
+-dontwarn javax.mail.Transport
+-dontwarn javax.mail.internet.AddressException
+-dontwarn javax.mail.internet.InternetAddress
+-dontwarn javax.mail.internet.MimeBodyPart
+-dontwarn javax.mail.internet.MimeMessage
+-dontwarn javax.mail.internet.MimeMultipart
+-dontwarn javax.script.ScriptEngine
+-dontwarn javax.script.ScriptEngineManager
+-dontwarn javax.servlet.ServletContextEvent
+-dontwarn javax.servlet.ServletContextListener
+-dontwarn javax.servlet.http.HttpServlet
+-dontwarn javax.swing.AbstractButton
+-dontwarn javax.swing.BorderFactory
+-dontwarn javax.swing.JButton
+-dontwarn javax.swing.JComponent
+-dontwarn javax.swing.JDialog
+-dontwarn javax.swing.JFrame
+-dontwarn javax.swing.JList
+-dontwarn javax.swing.JMenu
+-dontwarn javax.swing.JMenuBar
+-dontwarn javax.swing.JMenuItem
+-dontwarn javax.swing.JPanel
+-dontwarn javax.swing.JScrollPane
+-dontwarn javax.swing.JSplitPane
+-dontwarn javax.swing.JTextPane
+-dontwarn javax.swing.JViewport
+-dontwarn javax.swing.KeyStroke
+-dontwarn javax.swing.ListModel
+-dontwarn javax.swing.UIManager
+-dontwarn javax.swing.border.Border
+-dontwarn javax.swing.event.ListDataEvent
+-dontwarn javax.swing.event.ListDataListener
+-dontwarn javax.swing.event.ListSelectionListener
+-dontwarn javax.swing.text.JTextComponent
+-dontwarn org.apache.avalon.framework.logger.Logger
+-dontwarn org.apache.log.Hierarchy
+-dontwarn org.apache.log.Logger
+-dontwarn org.apache.xml.resolver.Catalog
+-dontwarn org.apache.xml.resolver.CatalogManager
+-dontwarn org.apache.xml.resolver.readers.CatalogReader
+-dontwarn org.apache.xml.resolver.readers.SAXCatalogReader

app/src/main/java/io/github/kabirnayeem99/islamqaorg/data/dataSource/IslamQaRemoteDataSource.kt

@@ -36,25 +36,30 @@ class IslamQaRemoteDataSource @Inject constructor(private val scrapingService: S
  */
  suspend fun getFiqhBasedQuestionsList(fiqh: Fiqh, pageNumber: Int): List<Question> {
 
- val qList = scrapingService.parseFiqhBasedQuestionsList(fiqh, pageNumber)
-
- if (qList.httpStatusCode != 200) throw Exception(qList.httpStatusMessage.ifBlank { "Failed to parse the questions." })
- if (qList.questionLinks.isEmpty() || qList.questions.isEmpty()) throw Exception("No questions were found.")
- if (qList.questions.size != qList.questionLinks.size) throw Exception("Failed to get answers for some questions")
-
- val questionAnswer = mutableListOf<Question>()
- qList.questions.forEachIndexed { index, question ->
- val answerLink = qList.questionLinks[index]
- questionAnswer.add(
- Question(
- question = question,
- url = answerLink,
- fiqh = fiqh.paramName,
+ try {
+ val qList =
+ scrapingService.parseFiqhBasedQuestionsList(fiqh, pageNumber) ?: return emptyList()
+
+ if (qList.httpStatusCode != 200) throw Exception(qList.httpStatusMessage.ifBlank { "Failed to parse the questions." })
+ if (qList.questionLinks.isEmpty() || qList.questions.isEmpty()) throw Exception("No questions were found.")
+ if (qList.questions.size != qList.questionLinks.size) throw Exception("Failed to get answers for some questions")
+
+ val questionAnswer = mutableListOf<Question>()
+ qList.questions.forEachIndexed { index, question ->
+ val answerLink = qList.questionLinks[index]
+ questionAnswer.add(
+ Question(
+ question = question,
+ url = answerLink,
+ fiqh = fiqh.paramName,
+ )
  )
- )
+ }
+ return questionAnswer.ifEmpty { throw Exception("Failed to parse links of the questions") }
+ } catch (e: Exception) {
+ Timber.e(e, "getFiqhBasedQuestionsList: ")
+ return emptyList()
  }
- return questionAnswer.ifEmpty { throw Exception("Failed to parse links of the questions") }
-
  }
 
  /**
@@ -64,25 +69,18 @@ class IslamQaRemoteDataSource @Inject constructor(private val scrapingService: S
  * @return A QuestionDetail object.
  */
  suspend fun getDetailedQuestionAndAnswer(url: String): QuestionDetail {
-
- Timber.d("Loading question answer of $url")
-
+ Timber.i("Loading question answer of $url")
  val dto = scrapingService.parseQuestionDetailScreen(url)
-
  val detail = QuestionDetail(
  questionTitle = dto.questionTitle,
  detailedQuestion = dto.detailedQuestion,
  detailedAnswer = dto.detailedAnswer,
  fiqh = dto.fiqh,
  source = dto.source,
  originalLink = dto.originalLink,
- nextQuestionLink = dto.nextQuestionLink,
- previousQuestionLink = dto.previousQuestionLink,
  relevantQuestions = dto.relevantQuestions,
  )
-
- Timber.d("getDetailedQuestionAndAnswer -> $detail")
-
+ Timber.i("getDetailedQuestionAndAnswer -> $detail")
  return detail
  }
 

app/src/main/java/io/github/kabirnayeem99/islamqaorg/data/dataSource/PreferenceDataSource.kt

@@ -98,6 +98,7 @@ class PreferenceDataSource @Inject constructor(private val context: Context) {
  val fiqh = getPreferredFiqh()
  return defaultPrefs.getInt(FIQH_LAST_PAGE + fiqh.paramName, 0)
  } catch (e: Exception) {
+ Timber.e("getCurrentFiqhLastPageSynced: ${e.localizedMessage}")
  return 0
  }
  }

app/src/main/java/io/github/kabirnayeem99/islamqaorg/data/dataSource/service/ScrapingService.kt

@@ -94,7 +94,7 @@ class ScrapingService {
  * @return A `QuestionDetailScreenDto` object.
  */
  suspend fun parseQuestionDetailScreen(link: String): QuestionDetailScreenDto {
- Timber.d("parseQuestionDetailScreen: link: $link")
+ Timber.i("parseQuestionDetailScreen: link: $link")
  return withContext(Dispatchers.IO) {
 
  val questionDetail = skrape(HttpFetcher) {
@@ -225,8 +225,7 @@ class ScrapingService {
  findAll {
  map {
  Question(
- question = it.text,
- url = it.eachHref.firstOrNull() ?: ""
+ question = it.text, url = it.eachHref.firstOrNull() ?: ""
  )
  }
  }
@@ -249,19 +248,27 @@ class ScrapingService {
  * @param pageNumber The page number of the questions list.
  * @return A list of questions.
  */
- suspend fun parseFiqhBasedQuestionsList(fiqh: Fiqh, pageNumber: Int): FiqhBasedQuestionListDto {
+ suspend fun parseFiqhBasedQuestionsList(
+ fiqh: Fiqh,
+ pageNumber: Int
+ ): FiqhBasedQuestionListDto? {
  return withContext(Dispatchers.Default) {
-
- val fiqhParamName = if (fiqh == Fiqh.UNKNOWN) Fiqh.HANAFI.paramName else fiqh.paramName
-
- val fiqhBasedQuestionUrl =
- "https://islamqa.org/category/${fiqhParamName}/page/$pageNumber/"
-
- val fiqhBasedQuestionListDto = skrape(HttpFetcher) {
- request { url = fiqhBasedQuestionUrl }
- response { getFiqhBasedQuestionListDtoOutOfResponse(this, fiqh) }
+ try {
+ val fiqhParamName =
+ if (fiqh == Fiqh.UNKNOWN) Fiqh.HANAFI.paramName else fiqh.paramName
+ Timber.i("Fiqh param name: $fiqhParamName")
+ val fiqhBasedQuestionUrl =
+ "https://islamqa.org/category/${fiqhParamName}/page/$pageNumber/"
+ Timber.i("fiqhBasedQuestionUrl: $fiqhBasedQuestionUrl")
+ val fiqhBasedQuestionListDto = skrape(HttpFetcher) {
+ request { url = fiqhBasedQuestionUrl }
+ response { getFiqhBasedQuestionListDtoOutOfResponse(this, fiqh) }
+ }
+ fiqhBasedQuestionListDto
+ } catch (e: Exception) {
+ Timber.e(e, "parseFiqhBasedQuestionsList: ${e.localizedMessage}")
+ null
  }
- fiqhBasedQuestionListDto
  }
  }
 
@@ -270,15 +277,20 @@ class ScrapingService {
  */
  fun getFiqhBasedQuestionListDtoOutOfResponse(
  result: Result, fiqh: Fiqh
- ): FiqhBasedQuestionListDto {
- result.apply {
- return FiqhBasedQuestionListDto(
- httpStatusCode = status { code },
- httpStatusMessage = status { message },
- questions = getFiqhBasedQuestionsFromResult(),
- questionLinks = getQuestionLinksForFiqhBasedQuestions(),
- fiqh = fiqh
- )
+ ): FiqhBasedQuestionListDto? {
+ try {
+ result.apply {
+ return FiqhBasedQuestionListDto(
+ httpStatusCode = status { code },
+ httpStatusMessage = status { message },
+ questions = getFiqhBasedQuestionsFromResult(),
+ questionLinks = getQuestionLinksForFiqhBasedQuestions(),
+ fiqh = fiqh
+ )
+ }
+ } catch (e: Exception) {
+ Timber.e(e, "getFiqhBasedQuestionListDtoOutOfResponse: ${e.localizedMessage}")
+ return null
  }
  }
 

app/src/main/java/io/github/kabirnayeem99/islamqaorg/data/dto/islamQa/FiqhBasedQuestionListDto.kt

@@ -14,8 +14,7 @@ import io.github.kabirnayeem99.islamqaorg.domain.entity.Fiqh
 data class FiqhBasedQuestionListDto(
  val httpStatusCode: Int,
  val httpStatusMessage: String,
- val questions
- : List<String>,
+ val questions: List<String>,
  val questionLinks: List<String>,
  val fiqh: Fiqh = Fiqh.HANAFI,
 )

app/src/main/java/io/github/kabirnayeem99/islamqaorg/data/repository/QuestionAnswerRepositoryImpl.kt

@@ -61,7 +61,7 @@ class QuestionAnswerRepositoryImpl
  }
 
  list.forEachIndexed { index, q ->
- delay(Random.nextLong(index * 100L))
+ delay(Random.nextLong((index + 1) * 100L))
  val questionDetailed = remoteDataSource.getDetailedQuestionAndAnswer(q.url)
  localDataSource.cacheQuestionDetail(questionDetailed)
  currentProgress++
@@ -72,7 +72,7 @@ class QuestionAnswerRepositoryImpl
  }
  return true
  } catch (e: Exception) {
- Timber.e("fetchAndSaveRandomQuestionList: ${e.localizedMessage}", e)
+ Timber.e(e, "fetchAndSaveRandomQuestionList: ${e.localizedMessage}", e)
  return false
  }
  }