diff --git a/src/ipa-sil.cpp b/src/ipa-sil.cpp
index fb2ef6a..821cbb7 100644
--- a/src/ipa-sil.cpp
+++ b/src/ipa-sil.cpp
@@ -9,58 +9,237 @@
 #include <fcitx/inputpanel.h>
 #include <fcitx/instance.h>
 
+#include <fcitx-utils/utf8.h>
+
+#include <codecvt>
+#include <locale>
+
+#include <unordered_map>
+
+// This is a giant map from a base character (stored as std::string because it's
+// easier that way) to a map. This new map maps SIL modifiers to the resulting
+// IPA character.
+// In the end, SILtoIPA["s"][FcitxKey_equal] == "ʃ", as `s=`
+// makes the IPA /ʃ/.
+static std::unordered_map<
+    std::string,
+    std::unordered_map<fcitx::KeySym const, std::string const> const> const
+    SILtoIPA = {
+        // ----- Consonants
+        {"t",
+         {
+             {FcitxKey_less, "ʈ"},
+         }},
+        {"d",
+         {
+             {FcitxKey_less, "ɖ"},
+         }},
+        {"j",
+         {
+             {FcitxKey_greater, "ʄ"},
+             {FcitxKey_less, "ʝ"},
+             {FcitxKey_equal, "ɟ"},
+         }},
+        {"g",
+         {
+             {FcitxKey_greater, "ɠ"},
+             {FcitxKey_less, "ɡ"},
+             {FcitxKey_equal, "ɣ"},
+         }},
+        {"G",
+         {
+             {FcitxKey_greater, "ʛ"},
+             {FcitxKey_equal, "ɢ"},
+         }},
+        {"?",
+         {
+             {FcitxKey_less, "ʕ"},
+             {FcitxKey_equal, "ʔ"},
+         }},
+        {"m",
+         {
+             {FcitxKey_greater, "ɱ"},
+         }},
+        {"n",
+         {
+             {FcitxKey_greater, "ŋ"},
+             {FcitxKey_less, "ɳ"},
+             {FcitxKey_equal, "ɲ"},
+         }},
+        {"N",
+         {
+             {FcitxKey_equal, "ɴ"},
+         }},
+        {"B",
+         {
+             {FcitxKey_greater, "ʁ"},
+             {FcitxKey_equal, "ʙ"},
+         }},
+        {"R",
+         {
+             {FcitxKey_less, "ɻ"},
+             {FcitxKey_equal, "ʀ"},
+         }},
+        {"v",
+         {
+             {FcitxKey_less, "ⱱ"},
+             {FcitxKey_equal, "ʋ"},
+         }},
+        {"r",
+         {
+             {FcitxKey_greater, "ɾ"},
+             {FcitxKey_equal, "ɹ"},
+         }},
+        {"f",
+         {
+             {FcitxKey_equal, "ɸ"},
+         }},
+        {"b",
+         {
+             {FcitxKey_greater, "ɓ"},
+             {FcitxKey_equal, "β"},
+         }},
+        {"t",
+         {
+             {FcitxKey_equal, "θ"},
+         }},
+        {"d",
+         {
+             {FcitxKey_greater, "ɗ"},
+             {FcitxKey_equal, "ð"},
+         }},
+        {"s",
+         {
+             {FcitxKey_less, "ʂ"},
+             {FcitxKey_equal, "ʃ"},
+         }},
+        {"z",
+         {
+             {FcitxKey_less, "ʐ"},
+             {FcitxKey_greater, "ʑ"},
+             {FcitxKey_equal, "ʒ"},
+         }},
+        {"c",
+         {
+             {FcitxKey_less, "ɕ"},
+             {FcitxKey_equal, "ç"},
+         }},
+        {"x",
+         {
+             {FcitxKey_equal, "χ"},
+         }},
+        {"h",
+         {
+             {FcitxKey_greater, "ħ"},
+             {FcitxKey_less, "ɦ"},
+             {FcitxKey_equal, "ɥ"},
+         }},
+        {"l",
+         {
+             {FcitxKey_greater, "ɮ"},
+             {FcitxKey_less, "ɭ"},
+             {FcitxKey_equal, "ɬ"},
+         }},
+        {"w",
+         {
+             {FcitxKey_greater, "ɰ"},
+             {FcitxKey_equal, "ʍ"},
+         }},
+        {"L",
+         {
+             {FcitxKey_greater, "ɺ"},
+             {FcitxKey_less, "ʎ"},
+             {FcitxKey_equal, "ʟ"},
+         }},
+        {"p",
+         {
+             {FcitxKey_equal, "ʘ"},
+         }},
+        {"!",
+         {
+             {FcitxKey_greater, "ǁ"},
+             {FcitxKey_equal, "ǂ"},
+         }},
+        {"y",
+         {
+             {FcitxKey_less, "ɥ"},
+             {FcitxKey_equal, "ʏ"},
+         }},
+        {"H",
+         {
+             {FcitxKey_greater, "ɧ"},
+             {FcitxKey_equal, "ʜ"},
+         }},
+        {"Q",
+         {
+             {FcitxKey_less, "ʢ"},
+             {FcitxKey_equal, "ʡ"},
+         }},
+
+        // ----- Vowels
+
+        {"I",
+         {
+             {FcitxKey_equal, "ɨ"},
+         }},
+        {"U",
+         {
+             {FcitxKey_equal, "ʉ"},
+         }},
+        {"u",
+         {
+             {FcitxKey_greater, "ʌ"},
+             {FcitxKey_equal, "ɯ"},
+         }},
+        {"i",
+         {
+             {FcitxKey_equal, "ɪ"},
+         }},
+        {"o",
+         {
+             {FcitxKey_less, "ɔ"},
+             {FcitxKey_greater, "ø"},
+         }},
+        {"E",
+         {
+             {FcitxKey_less, "œ"},
+             {FcitxKey_greater, "ɶ"},
+             {FcitxKey_equal, "ɘ"},
+         }},
+        {"O",
+         {
+             {FcitxKey_greater, "ɤ"},
+             {FcitxKey_equal, "ɵ"},
+         }},
+        {"e",
+         {
+             {FcitxKey_greater, "ɜ"},
+             {FcitxKey_less, "ɛ"},
+             {FcitxKey_equal, "ə"},
+         }},
+        {"O",
+         {
+             {FcitxKey_less, "ɞ"},
+         }},
+        {"a",
+         {
+             {FcitxKey_less, "æ"},
+             {FcitxKey_greater, "ɐ"},
+             {FcitxKey_equal, "ɑ"},
+         }},
+};
+
 std::optional<std::string> getIPAForSIL(std::string   baseCharacter,
                                         fcitx::KeySym silModifier)
 {
-	// TODO: Use a proper data structure, like a hash map
-	//       Even better: load that data off of a file!
-
-	// TODO: Add more characters! This is only enough for narrow RP
-	//       transcriptions...
-	if (baseCharacter == "s") {
-		if (silModifier == FcitxKey_equal) return "ʃ";
+	FCITX_INFO() << "Figuring IPA for " << baseCharacter << " & "
+	             << silModifier;
+	try {
+		return SILtoIPA.at(baseCharacter).at(silModifier);
+	} catch (std::out_of_range& e) {
+		FCITX_INFO() << "Couldn't get IPA character: " << e.what();
+		return {};
 	}
-
-	if (baseCharacter == "z") {
-		if (silModifier == FcitxKey_equal) return "ʒ";
-	}
-
-	if (baseCharacter == "t") {
-		if (silModifier == FcitxKey_equal) return "θ";
-	}
-
-	if (baseCharacter == "d") {
-		if (silModifier == FcitxKey_equal) return "ð";
-	}
-
-	if (baseCharacter == "n") {
-		if (silModifier == FcitxKey_greater) return "ŋ";
-	}
-
-	if (baseCharacter == "i") {
-		if (silModifier == FcitxKey_equal) return "ɪ";
-	}
-
-	if (baseCharacter == "e") {
-		if (silModifier == FcitxKey_equal) return "ə";
-	}
-
-	if (baseCharacter == "a") {
-		if (silModifier == FcitxKey_less) return "æ";
-		if (silModifier == FcitxKey_equal) return "ɑ";
-	}
-
-	if (baseCharacter == "o") {
-		if (silModifier == FcitxKey_less) return "ɔ";
-		if (silModifier == FcitxKey_equal) return "ɒ";
-	}
-
-	if (baseCharacter == "u") {
-		if (silModifier == FcitxKey_greater) return "ʌ";
-		if (silModifier == FcitxKey_less) return "ʊ";
-	}
-
-	return {};
 }
 
 bool isSILModifier(fcitx::KeySym key)
@@ -75,23 +254,20 @@ void SILState::handleAlphaKey(fcitx::Key key)
 	//       this!
 	if (key.check(FcitxKey_colon)) {
 		m_buffer.type("ː");
-		m_lastKey.reset();
 		return;
 	}
-	
-	if (!isSILModifier(key.sym()) || !m_lastKey.has_value()) {
-		m_lastKey = key.toString();
+
+	if (!isSILModifier(key.sym()) || getLastCharacter().empty()) {
 		m_buffer.type(key.sym());
 		return;
 	}
 
 	if (isSILModifier(key.sym())) {
-		auto ipaChar = getIPAForSIL(*m_lastKey, key.sym());
+		auto ipaChar = getIPAForSIL(getLastCharacter(), key.sym());
 		if (ipaChar) {
 			m_buffer.backspace();
 			m_buffer.type(*ipaChar);
 		}
-		m_lastKey.reset();
 	}
 }
 
@@ -110,9 +286,9 @@ void SILState::keyEvent(fcitx::KeyEvent& keyEvent)
 		reset();
 	} else if (keyEvent.key().check(FcitxKey_BackSpace))
 		m_buffer.backspace();
-	else if (keyEvent.key().isSimple())
+	else
 		handleAlphaKey(keyEvent.key());
-	//	m_buffer.type(keyEvent.key().sym());
+
 	updateUI();
 
 	keyEvent.filterAndAccept();
@@ -135,6 +311,15 @@ void SILState::updateUI()
 	m_ic->updatePreedit();
 }
 
+std::string SILState::getLastCharacter()
+{
+	// TODO: Use fcitx::InputBuffer::charAt once
+	// https://github.com/fcitx/fcitx5/issues/965 is fixed.
+	if (m_buffer.size() == 0) return "";
+	auto text = m_buffer.userInput();
+	return std::string(&text[m_buffer.cursorByChar() - 1], 1);
+}
+
 SILEngine::SILEngine(fcitx::Instance* instance)
     : m_factory(
           [this](fcitx::InputContext& ic) { return new SILState(this, &ic); })
diff --git a/src/ipa-sil.hpp b/src/ipa-sil.hpp
index 91134d9..a4e2a78 100644
--- a/src/ipa-sil.hpp
+++ b/src/ipa-sil.hpp
@@ -16,6 +16,8 @@
 #include <optional>
 #include <string>
 
+#include <fcitx-utils/utf8.h>
+
 class SILEngine;
 
 class SILState : public fcitx::InputContextProperty
@@ -37,12 +39,12 @@ class SILState : public fcitx::InputContextProperty
 		updateUI();
 	}
 
+	std::string getLastCharacter();
+
   private:
 	SILEngine*           m_engine;
 	fcitx::InputContext* m_ic;
 	fcitx::InputBuffer   m_buffer{{fcitx::InputBufferOption::NoOption}};
-
-	std::optional<std::string> m_lastKey{};
 };
 
 class SILEngine : public fcitx::InputMethodEngineV2