improve weapon list, mhx weapon scraper

scrapers/mhxg-weapons.py

@@ -4,6 +4,7 @@
 import urllib
 import os
 import json
+import sys
 
 from lxml import etree
 
@@ -17,6 +18,7 @@ _WEAPON_URLS = {
     "Sword and Shield": ["/data/1902.html", "/data/2884.html"],
     "Dual Blades": ["/data/1903.html", "/data/2885.html"],
     "Hammer": ["/data/1904.html", "/data/2886.html"],
+    "Hunting Horn": ["/data/1905.html", "/data/2887.html"],
     "Lance": ["/data/1906.html", "/data/2888.html"],
     "Gunlance": ["/data/1907.html", "/data/2889.html"],
     "Switch Axe": ["/data/1908.html", "/data/2890.html"],
@@ -95,6 +97,18 @@ _BOW_COATINGS = {
 }
 
 
+_HORN_NOTES = {
+    "#f3f3f3;": "W",
+    "blue;": "B",
+    "#e0002a;": "R",
+    "#ff00ff;": "P",
+    "#eeee00;": "Y",
+    "#00cc00;": "G",
+    "#00eeee;": "C",
+    "#ef810f;": "O",
+}
+
+
 def extract_weapon_list(wtype, tree):
     weapons = []
     rows = tree.xpath('//*[@id="sorter"]/tbody/tr')
@@ -102,26 +116,36 @@ def extract_weapon_list(wtype, tree):
     parent_href = None
     for row in rows:
         cells = list(row)
-        if len(cells) < 5:
-            continue
-        name, href, final = _parse_name_td(cells[0])
-        attack = int(cells[1].text)
-        affinity, defense, elements = _parse_elements_td(cells[2])
-        if wtype not in _RANGED_TYPES:
-            sharpness = _parse_sharpness_td(cells[-2])
+        if len(cells) == 4:
+            name, href, final = _parse_name_td(cells[0])
+            attack, affinity, defense, elements, slots = \
+                                            _parse_hh_attr_td(cells[1])
+            horn_notes = _parse_horn_notes_td(cells[2])
+            sharpness = _parse_sharpness_td(cells[3])
             shots, ammo = None, None
+        elif len(cells) in (5, 6):
+            name, href, final = _parse_name_td(cells[0])
+            attack = int(cells[1].text)
+            affinity, defense, elements = _parse_elements_td(cells[2])
+            if wtype not in _RANGED_TYPES:
+                sharpness = _parse_sharpness_td(cells[-2])
+                shots, ammo = None, None
+            else:
+                sharpness = [None, None]
+                if wtype == "Bow":
+                    shots, ammo = _parse_bow_td(cells[-2])
+            slots = _parse_slots_td(cells[-1])
+            horn_notes = None
         else:
-            sharpness = [None, None]
-            if wtype == "Bow":
-                shots, ammo = _parse_bow_td(cells[-2])
-        slots = _parse_slots_td(cells[-1])
+            continue
+
         data = dict(name_jp=name, name=name, wtype=wtype, final=final,
                     sharpness=sharpness[0], sharpness_plus=sharpness[1],
                     attack=attack, num_slots=slots,
                     affinity=affinity, defense=defense,
                     element=None, element_attack=None,
                     awaken=None, element_2=None, element_2_attack=None,
-                    phial=None, shelling_type=None, horn_notes=None,
+                    phial=None, shelling_type=None, horn_notes=horn_notes,
                     arc_type=None, ammo=ammo, shot_types=shots)
         if elements:
             data["element"] = elements[0][0]
@@ -179,6 +203,42 @@ def _parse_bow_td(td_element):
     return shots, coatings
 
 
+def _parse_horn_notes_td(td_element):
+    notes = []
+    note_spans = td_element.xpath("./div/span")
+    for span in note_spans:
+        style = span.attrib["style"]
+        color = style[len("color:"):]
+        notes.append(_HORN_NOTES[color])
+    return "".join(notes)
+
+
+def _parse_hh_attr_td(td_element):
+    spans = td_element.xpath('./span')
+    affinity = 0
+    defense = 0
+    attack = 0
+    elements = []
+    slots = 0
+    for span in spans:
+        span_class = span.attrib.get("class")
+        if span_class and span_class.startswith("type_"):
+            e = _parse_element(span.text.strip())
+            elements.append(e)
+        elif span_class and span_class == "b":
+            attack = int(span.text.strip())
+        else:
+            affinity = int(span.text.strip())
+    text_lines = td_element.text.strip().split("\n")
+    for line in text_lines:
+        if line.startswith(u"防御+"):
+            defense = int(line[3:])
+
+    if td_element.tail:
+        slots = td_element.tail.count(u"◯")
+    return attack, affinity, defense, elements, slots
+
+
 def _parse_elements_td(td_element):
     # 会心<span class="c_r">-20</span>%<br>
     # 防御+5

web/js/common.js

@@ -299,14 +299,41 @@ function set_bow_values(weapon_data) {
 }
 
 
-function set_horn_melodies_title(weapon_data) {
+function set_horn_melodies_title(weapon_data, melody_map) {
     if (! weapon_data["horn_notes"]) {
-        weapon_data["horn_melodies_title"] = ""
+        weapon_data["horn_melodies_title"] = "";
+        return;
+    }
+
+    var notes = weapon_data["horn_notes"];
+
+    var melodies;
+    if (melody_map) {
+        melodies = melody_map[notes];
+        if (! melodies) {
+            // Try flipping second two notes if not found, mhx data is
+            // either wrong or game itself is inconsistant about note
+            // order.
+            notes = notes.substring(0, 1) + notes.substring(2, 3)
+                    + notes.substring(1, 2);
+            melodies = melody_map[notes];
+            if (melodies) {
+                weapon_data["horn_notes"] = notes;
+            }
+        }
+    } else {
+        melodies = weapon_data["horn_melodies"];
+    }
+
+    if (! melodies) {
+        var msg = "Unknown melodies for " + notes;
+        weapon_data["horn_melodies_title"] = msg;
+        console.log(msg);
         return;
     }
 
     var lines = [];
-    $.each(weapon_data["horn_melodies"], function(i, melody) {
+    $.each(melodies, function(i, melody) {
         var space = Array(6 - melody["song"].length).join(" ");
         lines.push(melody["song"] + space + melody["effect1"]);
     });
@@ -402,7 +429,7 @@ function cmp_arrays(alist, blist) {
     for (var i=0; i<alist.length; i++) {
         a = alist[i];
         b = blist[i];
-        if (a == null && b == null) {
+        if (a == null || b == null) {
             // ignore
         } else if (typeof a == "object") {
             cmp = cmp_arrays(a, b);

web/mhx/weaponlist.html

@@ -99,6 +99,7 @@
 
   <script type="text/javascript">
     var WEAPON_LIST = null;
+    var HORN_MELODY_MAP = null;
 
     $.ajax({
         url: "/jsonapi/mhx/weapon_list.json",
@@ -110,6 +111,16 @@
         }
     });
 
+    $.ajax({
+        url: "/jsonapi/horn_melody/_index_notes.json",
+        async: false,
+        dataType: "json",
+        success: function (data) {
+            HORN_MELODY_MAP = data;
+        }
+    });
+
+
     var template_row = new EJS({ url: "/templates/weaponrow.ejs" });
 
     $(document).ready(function(){
@@ -236,7 +247,7 @@
                 match_count += 1;
                 set_sharpness_titles(weapon_data);
                 set_bow_values(weapon_data);
-                set_horn_melodies_title(weapon_data);
+                set_horn_melodies_title(weapon_data, HORN_MELODY_MAP);
                 weapon_data["wtype_short"] =
                           WEAPON_TYPE_ABBR[weapon_data["wtype"]];
                 weapon_data["ELEMENT_ABBR"] = ELEMENT_ABBR;

web/templates/weaponrow.ejs

@@ -71,5 +71,5 @@
   title="<%= bow_coatings_text %>"><%= bow_shots_text %></td>
 <% } %>
 <td title="<%= horn_melodies_title %>"
-    style="text-align:right"><%= horn_notes %></td>
+    style="text-align:right; font-family:mono"><%= horn_notes %></td>
 </tr>

web/weaponlist.html

@@ -221,7 +221,10 @@
     function update_weapon_list(state) {
         var match_count = 0;
         var cp_setups = [];
-        var comps = state["weapon_component_text"].split("|");
+        var comps = state["weapon_component_text"];
+        if (comps) {
+            comps = comps.split("|");
+        }
         console.log("updating weapon list: " + JSON.stringify(state));
         var results = [];
         $.each(WEAPON_ID_IDX, function(weapon_id, weapon_list) {