From c06482e3406e838fb572c4b9ec82fa855b426e5a Mon Sep 17 00:00:00 2001 From: Tom Ritter Date: Wed, 18 Oct 2023 14:13:09 -0400 Subject: [PATCH] Bug 1858193: Add a OneTimeWorkRequest to collect Telemetry for Installed Fonts --- app/metrics.yaml | 19 ++ app/pings.yaml | 13 + .../java/org/mozilla/fenix/HomeActivity.kt | 2 + .../metrics/fonts/FontEnumerationWorker.kt | 223 ++++++++++++++++ .../components/metrics/fonts/FontParser.kt | 245 ++++++++++++++++++ .../java/org/mozilla/fenix/utils/Settings.kt | 8 + app/src/main/res/values/preference_keys.xml | 2 + 7 files changed, 512 insertions(+) create mode 100644 app/src/main/java/org/mozilla/fenix/components/metrics/fonts/FontEnumerationWorker.kt create mode 100644 app/src/main/java/org/mozilla/fenix/components/metrics/fonts/FontParser.kt diff --git a/app/metrics.yaml b/app/metrics.yaml index 595135984f..2c5d2f5b70 100644 --- a/app/metrics.yaml +++ b/app/metrics.yaml @@ -2612,6 +2612,25 @@ metrics: metadata: tags: - Experiments + font_list_json: + type: text + lifetime: ping + description: | + A JSON blob representing the installed fonts + send_in_pings: + - font-list + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1858193 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1858193#c2 + data_sensitivity: + # Text metrics are _required_ to be web_activity or highly_sensitive, so even though this + # is more like 'technical' (per the Data Review), I'm marking highly sensitive. + - highly_sensitive + notification_emails: + - android-probes@mozilla.com + - tom@mozilla.com + expires: 122 customize_home: most_visited_sites: diff --git a/app/pings.yaml b/app/pings.yaml index 24f16f8972..1ffe0dffb5 100644 --- a/app/pings.yaml +++ b/app/pings.yaml @@ -77,6 +77,7 @@ cookie-banner-report-site: - https://github.com/mozilla-mobile/firefox-android/pull/1298#pullrequestreview-1350344223 notification_emails: - android-probes@mozilla.com + fx-suggest: description: | A ping representing a single event occurring with or to a Firefox Suggestion. @@ -91,3 +92,15 @@ fx-suggest: - lina@mozilla.com - ttran@mozilla.com - najiang@mozilla.com + +font-list: + description: | + List of fonts installed on the user's device + include_client_id: false + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1858193 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1858193#c2 + notification_emails: + - android-probes@mozilla.com + - tom@mozilla.com diff --git a/app/src/main/java/org/mozilla/fenix/HomeActivity.kt b/app/src/main/java/org/mozilla/fenix/HomeActivity.kt index 69d8ffb8d9..494c5934f3 100644 --- a/app/src/main/java/org/mozilla/fenix/HomeActivity.kt +++ b/app/src/main/java/org/mozilla/fenix/HomeActivity.kt @@ -98,6 +98,7 @@ import org.mozilla.fenix.browser.browsingmode.DefaultBrowsingModeManager import org.mozilla.fenix.components.appstate.AppAction import org.mozilla.fenix.components.metrics.BreadcrumbsRecorder import org.mozilla.fenix.components.metrics.GrowthDataWorker +import org.mozilla.fenix.components.metrics.fonts.FontEnumerationWorker import org.mozilla.fenix.databinding.ActivityHomeBinding import org.mozilla.fenix.exceptions.trackingprotection.TrackingProtectionExceptionsFragmentDirections import org.mozilla.fenix.experiments.ResearchSurfaceDialogFragment @@ -521,6 +522,7 @@ open class HomeActivity : LocaleAwareAppCompatActivity(), NavHostActivity { } GrowthDataWorker.sendActivatedSignalIfNeeded(applicationContext) + FontEnumerationWorker.sendActivatedSignalIfNeeded(applicationContext) ReEngagementNotificationWorker.setReEngagementNotificationIfNeeded(applicationContext) MessageNotificationWorker.setMessageNotificationWorker(applicationContext) } diff --git a/app/src/main/java/org/mozilla/fenix/components/metrics/fonts/FontEnumerationWorker.kt b/app/src/main/java/org/mozilla/fenix/components/metrics/fonts/FontEnumerationWorker.kt new file mode 100644 index 0000000000..00a953d84a --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/components/metrics/fonts/FontEnumerationWorker.kt @@ -0,0 +1,223 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.components.metrics.fonts + +import android.content.Context +import android.content.res.Configuration +import android.graphics.fonts.Font +import android.graphics.fonts.SystemFonts +import android.os.Build +import android.os.LocaleList +import androidx.work.BackoffPolicy +import androidx.work.CoroutineWorker +import androidx.work.ExistingWorkPolicy +import androidx.work.OneTimeWorkRequest +import androidx.work.WorkManager +import androidx.work.WorkerParameters +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import mozilla.components.service.glean.private.CommonMetricData +import mozilla.components.service.glean.private.Lifetime +import mozilla.components.service.glean.private.TextMetricType +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject +import org.mozilla.fenix.Config +import org.mozilla.fenix.ext.settings +import java.io.File +import java.util.Locale +import java.util.concurrent.TimeUnit +import kotlin.time.Duration.Companion.hours + +/** + * Parse all of the fonts on the user's phone, then put them into the + * `font_list_json` Metric to be submitted via Telemetry later. + */ +class FontEnumerationWorker( + context: Context, + workerParameters: WorkerParameters, +) : CoroutineWorker(context, workerParameters) { + @Suppress("TooGenericExceptionCaught") + override suspend fun doWork(): Result = withContext(Dispatchers.IO) { + val s: String + try { + readAllFonts() + s = createJSONString() + } catch (e: Exception) { + return@withContext Result.retry() + } + + // We only want to get one submission per user, so set lifetime=Ping, + // And the value will be only set once (when this WorkRequest is run) + val fontList = + TextMetricType( + CommonMetricData( + category = "metrics", + name = "font_list_json", + sendInPings = listOf("font-list"), + lifetime = Lifetime.PING, + disabled = false, + ), + ) + fontList.set(s) + + // To avoid getting multiple submissions from new installs, set directly + // to the desired number of submissions + applicationContext.settings().numFontListSent = kDesiredSubmissions + + return@withContext Result.success() + } + + private val brokenFonts: ArrayList> = ArrayList() + private val fonts: MutableSet = HashSet() + + @Suppress("TooGenericExceptionCaught") + private fun readAllFonts() { + for (path in getSystemFonts()) { + try { + fonts.add(FontParser.parse(path)) + } catch (e: Exception) { + brokenFonts.add(Pair(path, FontParser.calculateFileHash(path))) + } + } + for (path in getAPIFonts()) { + try { + fonts.add(FontParser.parse(path)) + } catch (e: Exception) { + brokenFonts.add(Pair(path, FontParser.calculateFileHash(path))) + } + } + } + + /** + * This function creates a single JSON String containing + * The user's phone information, as well as all the fonts and their information, + * And the names of files that encountered a parsing error. + */ + @Throws(JSONException::class) + fun createJSONString(): String { + val submission = JSONObject() + + run { + submission.put("brand", Build.BRAND) + submission.put("device", Build.DEVICE) + submission.put("hardware", Build.HARDWARE) + submission.put("manufacturer", Build.MANUFACTURER) + submission.put("model", Build.MODEL) + submission.put("product", Build.PRODUCT) + submission.put("release_version", Build.VERSION.RELEASE) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + submission.put("security_patch", Build.VERSION.SECURITY_PATCH) + submission.put("base_os", Build.VERSION.BASE_OS) + } else { + submission.put("security_patch", "too-low-version") + submission.put("base_os", "too-low-version") + } + val config: Configuration = this.applicationContext.resources.configuration + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + val supportedLocales: LocaleList = LocaleList.getDefault() + val sb = StringBuilder() + for (i in 0 until supportedLocales.size()) { + val locale: Locale = supportedLocales.get(i) + sb.append(locale.toString()) + sb.append(",") + } + submission.put("current_locale", config.locales[0].toString()) + submission.put("all_locales", sb.toString()) + } else { + @Suppress("DEPRECATION") + submission.put("current_locale", config.locale.toString()) + submission.put("all_locales", "too-low-version") + } + } + + val fontArr = JSONArray() + for (fontDetails in fonts) { + fontArr.put(fontDetails.toJson()) + } + + val errorArr = JSONArray() + for (error in brokenFonts) { + val errorObj = JSONObject() + errorObj.put("path", error.first) + errorObj.put("hash", error.second) + errorArr.put(errorObj) + } + + submission.put("fonts", fontArr) + submission.put("errors", errorArr) + + return submission.toString() + } + + companion object { + private const val FONT_ENUMERATOR_WORK_NAME = "org.mozilla.fenix.metrics.font.work" + private val HOUR_MILLIS: Long = 1.hours.inWholeMilliseconds + private const val SIX: Long = 6 + + /** + * Schedules the Activated User event if needed. + */ + fun sendActivatedSignalIfNeeded(context: Context) { + val instanceWorkManager = WorkManager.getInstance(context) + + if (!Config.channel.isNightlyOrDebug) { + return + } + + if (context.settings().numFontListSent >= kDesiredSubmissions) { + return + } + + val fontEnumeratorWork = + OneTimeWorkRequest.Builder(FontEnumerationWorker::class.java) + .setInitialDelay(HOUR_MILLIS, TimeUnit.MILLISECONDS) + .setBackoffCriteria(BackoffPolicy.EXPONENTIAL, SIX, TimeUnit.HOURS) + .build() + + instanceWorkManager.beginUniqueWork( + FONT_ENUMERATOR_WORK_NAME, + ExistingWorkPolicy.KEEP, + fontEnumeratorWork, + ).enqueue() + } + + private fun getSystemFonts(): ArrayList { + val file = File("/system/fonts") + val ff: Array? = file.listFiles() + val systemFonts: ArrayList = ArrayList() + if (ff != null) { + for (f in ff) { + systemFonts.add(f.absolutePath) + } + } + return systemFonts + } + + private fun getAPIFonts(): List { + val aPIFonts: List + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { + aPIFonts = emptyList() + } else { + aPIFonts = ArrayList() + val apiFonts: Set = SystemFonts.getAvailableFonts() + for (f in apiFonts) { + f.file?.let { + aPIFonts.add(it.absolutePath) + } + } + } + return aPIFonts + } + + /** + * The number of font submissions we would like from a user. + * We will increment this number by one (via a code patch) when + * we wish to perform another data collection effort on the Nightly + * population. + */ + const val kDesiredSubmissions: Int = 1 + } +} diff --git a/app/src/main/java/org/mozilla/fenix/components/metrics/fonts/FontParser.kt b/app/src/main/java/org/mozilla/fenix/components/metrics/fonts/FontParser.kt new file mode 100644 index 0000000000..31a2901136 --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/components/metrics/fonts/FontParser.kt @@ -0,0 +1,245 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.components.metrics.fonts + +import org.json.JSONException +import org.json.JSONObject +import java.io.DataInputStream +import java.io.FileInputStream +import java.io.IOException +import java.io.InputStream +import java.security.MessageDigest +import java.security.NoSuchAlgorithmException +import kotlin.math.min + +/** + * FontMetric represents the information about a Font File + */ +data class FontMetric( + val path: String = "", + val hash: String = "", +) { + var family: String = "" + var subFamily: String = "" + var uniqueSubFamily: String = "" + var fullName: String = "" + var fontVersion: String = "" + var revision: Int = -1 + var created: Long = -1L + var modified: Long = -1L + + /** + * Return a JSONObject of this Font's details + */ + fun toJson(): JSONObject { + val jsonObject = JSONObject() + try { + // Use abbreviations to make the json smaller + jsonObject.put("F", family.replace("\u0000", "")) + jsonObject.put("SF", subFamily.replace("\u0000", "")) + jsonObject.put("USF", uniqueSubFamily.replace("\u0000", "")) + jsonObject.put("FN", fullName.replace("\u0000", "")) + jsonObject.put("V", fontVersion.replace("\u0000", "")) + jsonObject.put("R", revision) + jsonObject.put("C", created) + jsonObject.put("M", modified) + jsonObject.put("H", hash) + jsonObject.put("P", path.replace("\u0000", "")) + } catch (_: JSONException) { + } + return jsonObject + } +} + +/** + * Parse a font, given via an InputStream, to extract the Font information + * including Family, SubFamily, Revision, etc + */ +object FontParser { + /** + * Parse a font file and return a FontMetric object describing it + */ + fun parse(path: String): FontMetric { + return parse(path, FileInputStream(path)) + } + + /** + * Parse a font file and return a FontMetric object describing it + */ + fun parse(path: String, inputStream: InputStream): FontMetric { + val hash = calculateFileHash(inputStream) + val fontDetails = FontMetric(path, hash) + + inputStream.reset() + readFontFile(inputStream, fontDetails) + + return fontDetails + } + + @Suppress("MagicNumber") + private fun readFontFile(inputStream: InputStream, fontDetails: FontMetric) { + val file = DataInputStream(inputStream) + val numFonts: Int + val magicNumber = file.readInt() + var bytesReadSoFar = 4 + + if (magicNumber == 0x74746366) { + // The Font File has a TTC Header + val majorVersion = file.readUnsignedShort() + file.skipBytes(2) // Minor Version + numFonts = file.readInt() + bytesReadSoFar += 8 + + file.skipBytes(4 * numFonts) // OffsetTable + bytesReadSoFar += 4 * numFonts + if (majorVersion == 2) { + file.skipBytes(12) + bytesReadSoFar += 12 + } + + file.skipBytes(4) // Magic Number for the Font + bytesReadSoFar += 4 + } + val numTables: Int = file.readUnsignedShort() + bytesReadSoFar += 2 + file.skipBytes(6) // Rest of header + bytesReadSoFar += 6 + + // Find the head table + var headOffset = 0 + var nameOffset = 0 + var nameLength = 0 + for (i in 0 until numTables) { + val tableName = + CharArray(4) { + file.readUnsignedByte().toChar() + } + file.skipBytes(4) // checksum + val offset = file.readInt() // technically it's unsigned but we should be okay + val length = file.readInt() // technically it's unsigned but we should be okay + + bytesReadSoFar += 16 + + if (String(tableName) == "head") { + headOffset = offset + } else if (String(tableName) == "name") { + nameOffset = offset + nameLength = length + } + } + + if (headOffset == 0 || nameOffset == 0) { + throw IOException("Could not find head or name table") + } + + if (headOffset < nameOffset) { + file.skipBytes(headOffset - bytesReadSoFar) + bytesReadSoFar = headOffset + bytesReadSoFar += readHeadTable(file, fontDetails) + file.skipBytes(nameOffset - bytesReadSoFar) + readNameTable(file, nameLength, fontDetails) + } else { + file.skipBytes(nameOffset - bytesReadSoFar) + bytesReadSoFar = nameOffset + bytesReadSoFar += readNameTable(file, nameLength, fontDetails) + file.skipBytes(headOffset - bytesReadSoFar) + readHeadTable(file, fontDetails) + } + file.close() + } + + @Suppress("MagicNumber") + private fun readHeadTable(file: DataInputStream, fontDetails: FontMetric): Int { + // Find the details in the head table + file.skipBytes(4) // Fixed version + fontDetails.revision = file.readInt() + file.skipBytes(12) // checksum, magic, flags, units + fontDetails.created = file.readLong() + fontDetails.modified = file.readLong() + return 36 + } + + @Suppress("MagicNumber") + private fun readNameTable( + file: DataInputStream, + tableLength: Int, + fontDetails: FontMetric, + ): Int { + file.skipBytes(2) // format + val numNames = file.readUnsignedShort() + val stringOffset = file.readUnsignedShort() + var bytesReadSoFar = 6 + val nameTable = arrayListOf>() + + for (i in 0 until numNames) { + file.skipBytes(6) // platform id, encoding id, langid + val nameID = file.readUnsignedShort() + val length = file.readUnsignedShort() + val offset = file.readUnsignedShort() + nameTable.add(Triple(nameID, length, offset)) + bytesReadSoFar += 12 + } + + val stringTableSize = min(tableLength - bytesReadSoFar, tableLength - stringOffset) + val stringTable = ByteArray(stringTableSize) + + if (stringTable.size != file.read(stringTable)) { + throw IOException("Did not read entire string table") + } + + bytesReadSoFar += stringTable.size + + // Now we're at the beginning of the string table + for (i in nameTable) { + when (i.first) { + 1 -> fontDetails.family = getString(stringTable, i.third, i.second) + 2 -> fontDetails.subFamily = getString(stringTable, i.third, i.second) + 3 -> fontDetails.uniqueSubFamily = getString(stringTable, i.third, i.second) + 4 -> fontDetails.fullName = getString(stringTable, i.third, i.second) + 5 -> fontDetails.fontVersion = getString(stringTable, i.third, i.second) + } + } + return bytesReadSoFar + } + + private fun getString( + stringTable: ByteArray, + offset: Int, + length: Int, + ): String { + return String(stringTable.copyOfRange(offset, offset + length)) + } + + /** + * Calculate the SHA-256 hash of the file passed + */ + fun calculateFileHash(path: String): String { + return calculateFileHash(FileInputStream(path)) + } + + /** + * Calculate the SHA-256 hash of the InputStream passed + */ + @Suppress("MagicNumber") + private fun calculateFileHash(inputStream: InputStream): String { + try { + val md = MessageDigest.getInstance("SHA-256") + val buffer = ByteArray(8192) + var bytesRead: Int + while (inputStream.read(buffer).also { bytesRead = it } != -1) { + md.update(buffer, 0, bytesRead) + } + val digest = md.digest() + // Convert the byte array to a hexadecimal string + val hashBuilder = StringBuilder() + for (b in digest) { + hashBuilder.append(String.format("%02X", b)) + } + return hashBuilder.toString() + } catch (_: NoSuchAlgorithmException) { + return "sha-256-not-found" + } + } +} diff --git a/app/src/main/java/org/mozilla/fenix/utils/Settings.kt b/app/src/main/java/org/mozilla/fenix/utils/Settings.kt index 6f43974113..c5a0a6402b 100644 --- a/app/src/main/java/org/mozilla/fenix/utils/Settings.kt +++ b/app/src/main/java/org/mozilla/fenix/utils/Settings.kt @@ -1866,6 +1866,14 @@ class Settings(private val appContext: Context) : PreferencesHolder { default = false, ) + /** + * Font List Telemetry Ping Sent + */ + var numFontListSent by intPreference( + key = appContext.getPreferenceKey(R.string.pref_key_num_font_list_sent), + default = 0, + ) + /** * Indicates how many days in the first week user opened the app. */ diff --git a/app/src/main/res/values/preference_keys.xml b/app/src/main/res/values/preference_keys.xml index d1debd6b16..27100b8ec8 100644 --- a/app/src/main/res/values/preference_keys.xml +++ b/app/src/main/res/values/preference_keys.xml @@ -300,6 +300,8 @@ pref_key_show_collections_home + pref_key_num_font_list_sent + pref_key_growth_user_activated_sent pref_key_growth_early_browse_count