add mhx weapon search, hammer data only

.gitignore

@@ -1,3 +1,4 @@
 *.pyc
 README.html
 RECOMMENDATIONS.html
+tmp/

scrapers/_pathfix.py

@@ -0,0 +1,15 @@
+"""
+Hack to get scripts to run from source checkout without having to set
+PYTHONPATH.
+"""
+
+import sys
+from os.path import dirname, join, abspath
+
+bin_path = dirname(__file__)
+project_path = abspath(join(bin_path, ".."))
+sys.path.insert(0, project_path)
+
+db_path = join(project_path, "db", "mh4u.db")
+motion_values_path = join(project_path, "db", "motion_values.json")
+web_path = join(project_path, "web")

scrapers/mhxg-weapons.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf8 :
+
+import urllib
+import os
+import json
+
+from lxml import etree
+
+import _pathfix
+
+
+_WEAPON_URLS = {
+    "Hammer": ["http://wiki.mhxg.org/data/1904.html",
+               "http://wiki.mhxg.org/data/2886.html"],
+}
+
+
+_ELEMENT_MAP = {
+    u"火": "Fire",
+    u"水": "Water",
+    u"雷": "Thunder",
+    u"氷": "Ice",
+    u"龍": "Dragon",
+    u"毒": "Poison",
+    u"麻痺": "Paralysis",
+    u"睡眠": "Sleep",
+    u"爆破": "Blast",
+}
+
+
+def extract_weapon_list(wtype, tree):
+    weapons = []
+    rows = tree.xpath('//*[@id="sorter"]/tbody/tr')
+    i = 0
+    parent_name = None
+    parent_href = None
+    for row in rows:
+        cells = list(row)
+        if len(cells) != 5:
+            continue
+        name, href, final = _parse_name_td(cells[0])
+        attack = int(cells[1].text)
+        affinity, defense, element, element_attack = _parse_extra_td(cells[2])
+        sharpness = _parse_sharpness_td(cells[3])
+        slots = _parse_slots_td(cells[4])
+        data = dict(name_jp=name, name=name, wtype=wtype, final=final,
+                    sharpness=sharpness[0], sharpness_plus=sharpness[1],
+                    attack=attack, num_slots=slots,
+                    affinity=affinity, defense=defense,
+                    element=element, element_attack=element_attack)
+        if href is None or href == parent_href:
+            data["parent"] = parent_name
+            data["href"] = parent_href
+        else:
+            data["href"] = href
+            data["parent"] = None
+            parent_name = name
+            parent_href = href
+        data["url"] = "http://wiki.mhxg.org" + data["href"]
+        weapons.append(data)
+    return weapons
+
+
+def _parse_extra_td(td_element):
+    # 会心<span class="c_r">-20</span>%<br>
+    # 防御+5
+    spans = td_element.xpath('./span')
+    affinity = 0
+    defense = 0
+    element = None
+    element_attack = None
+    for span in spans:
+        span_class = span.attrib.get("class")
+        if span_class and span_class.startswith("type_"):
+            element, element_attack = _parse_element(span.text.strip())
+        else:
+            affinity = int(span.text.strip())
+    text_lines = td_element.text.strip().split("\n")
+    for line in text_lines:
+        if line.startswith(u"防御+"):
+            defense = int(line[3:])
+    return affinity, defense, element, element_attack
+
+
+def _parse_element(text):
+    for jp_element in sorted(_ELEMENT_MAP.keys(), key=lambda s: len(s),
+                             reverse=True):
+        if text.startswith(jp_element):
+            value = int(text[len(jp_element):])
+            element = _ELEMENT_MAP[jp_element]
+            return element, value
+    raise ValueError("Bad element: %r" % text)
+
+
+def _parse_name_td(td_element):
+    href = None
+    name = None
+    final = 0
+    links = td_element.xpath('.//a')
+    if links:
+        assert len(links) == 1
+        href = links[0].attrib["href"]
+        name = links[0].text
+        style = td_element.xpath('./span[@class="b"]/@style')
+        if style and "background-color:#eec3e6;" in style:
+            final = 1
+    else:
+        img_element = td_element.xpath('./img')[0]
+        name = img_element.tail.strip()
+    if name.endswith("10"):
+        final = 1
+    return name, href, final
+
+
+def _parse_slots_td(td_element):
+    text = td_element.text
+    if text:
+        return text.count(u"◯")
+    return 0
+
+
+def _parse_sharpness_td(td_element):
+    div = td_element[0]
+    subs = list(div)
+    sharpness, sharpness_plus = [], []
+    current = sharpness
+    for sub in subs:
+        if sub.tag == "br":
+            current = sharpness_plus
+            continue
+        assert sub.tag == "span", sub.tag
+        if sub.attrib["class"] == "kr7":
+            continue
+        if sub.text is None:
+            continue
+        current.append(sub.text.count("."))
+    for i in xrange(len(sharpness), 6):
+        sharpness.append(0)
+    for i in xrange(len(sharpness_plus), 6):
+        sharpness_plus.append(0)
+    return sharpness, sharpness_plus
+
+
+def _main():
+    tmp_path = os.path.join(_pathfix.project_path, "tmp")
+    weapon_list = []
+    parser = etree.HTMLParser()
+    for wtype, urls in _WEAPON_URLS.iteritems():
+        for i, url in enumerate(urls):
+            fpath = os.path.join(tmp_path, "%s-%d.html" % (wtype, i))
+            urllib.urlretrieve(url, fpath)
+            with open(fpath) as f:
+                tree = etree.parse(f, parser)
+                wlist = extract_weapon_list(wtype, tree)
+            weapon_list.extend(wlist)
+    print json.dumps(weapon_list, indent=2)
+
+
+if __name__ == '__main__':
+    _main()

web/mhx/weaponlist.html

@@ -0,0 +1,353 @@
+<html>
+<head>
+  <title>Poogie's Weapon List</title>
+
+  <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
+
+  <link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.11.3/themes/smoothness/jquery-ui.css" />
+  <script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.11.3/jquery-ui.min.js"></script>
+
+  <script type="text/javascript" src="/js/ejs_production.js"></script>
+
+  <script type="text/javascript" src="/js/common.js"></script>
+
+  <style>
+      label {
+          font-weight: bold;
+      }
+
+      body {
+          font-family: sans, sans-serif;
+      }
+
+      td.plus {
+          background-color: LightCyan;
+      }
+
+      td.minus {
+          background-color: LightPink;
+      }
+
+      td.num {
+          text-align: right;
+      }
+
+      /*@media (max-width: 600) {*/
+      .flexbox {
+          display: flex;
+          flex-direction: row;
+          flex-wrap: wrap;
+      }
+
+      .sharpness-bar {
+          border: 1px #d3d3d3 solid;
+          min-width: 72px;
+          height: 10px;
+          background-color: #d3d3d3;
+          float: left;
+          clear: both;
+      }
+
+      .sharpness-bar span {
+          display: inline-block;
+          height: 100%;
+          float: left;
+      }
+
+      .sharpness-bar .red {
+          background-color: #C00C38 !important;
+      }
+
+      .sharpness-bar .orange {
+          background-color: #E85018 !important;
+      }
+
+      .sharpness-bar .yellow {
+          background-color: #F0C830 !important;
+      }
+
+      .sharpness-bar .green {
+          background-color: #58D000 !important;
+      }
+
+      .sharpness-bar .blue {
+          background-color: #3068E8 !important;
+      }
+
+      .sharpness-bar .white {
+          background-color: #F0F0F0 !important;
+      }
+
+      .sharpness-bar .purple {
+          /* no purple in mhx */
+          display: none;
+      }
+
+      #sharpness_popup {
+          position: absolute;
+          display: none;
+          border: 1px solid;
+          background: rgba(204, 204, 204, 0.9);
+          z-index: 10;
+      }
+
+      #cp_div {
+          display: none;
+      }
+
+  </style>
+
+  <script type="text/javascript">
+    var WEAPON_LIST = null;
+
+    $.ajax({
+        url: "/jsonapi/mhx/weapon_list.json",
+        async: false,
+        dataType: "json",
+        success: function (data) {
+            WEAPON_LIST = data;
+            console.log("weapon count " + WEAPON_LIST.length);
+        }
+    });
+
+    var template_row = new EJS({ url: "/templates/weaponrow.ejs" });
+
+    $(document).ready(function(){
+        init_page();
+
+        $("#sharpness_popup").on("click", function(evt) {
+            $(this).html("").offset({top:0, left:0}).hide();
+        });
+
+        $("#weapon_table").on("click", "#sharpness_td", function(evt) {
+            var td_obj = $(evt.currentTarget);
+            var offset = td_obj.offset();
+            var sharpness = td_obj.data("sharpness");
+            $("#sharpness_popup").html(sharpness).offset(offset).show();
+        });
+    });
+
+    function init_page() {
+        var qs = load_qs();
+        $(window).on("popstate", function(e) {
+            var oe = e.originalEvent;
+            if (oe.state !== null) {
+                console.log("popState:" + JSON.stringify(oe.state));
+                update_weapon_list(oe.state);
+            }
+        });
+        $("#search").click(function(evt) {
+            var state = get_ui_state();
+            save_state(state);
+            update_weapon_list(state);
+        });
+
+        if (qs) {
+            update_weapon_list(qs);
+        }
+    }
+
+    function load_qs() {
+        if ($.QueryString["weapon_type"]) {
+            load_state($.QueryString);
+            return $.QueryString;
+        }
+        return null;
+    }
+
+    function get_ui_state() {
+        return { "weapon_type": $("#weapon_type").val(),
+                 "weapon_element": $("#weapon_element").val(),
+                 "weapon_final": $("#weapon_final").is(":checked"),
+                 "weapon_name_text": $("#weapon_name_text").val() };
+    }
+
+    function load_state(state) {
+        $("#weapon_type").val(state["weapon_type"]);
+        $("#weapon_element").val(state["weapon_element"]);
+        $("#weapon_final").prop("checked", state["weapon_final"]);
+        $("#weapon_name_text").val(state["weapon_name_text"]);
+    }
+
+    function save_state(state, replace) {
+        var url = "weaponlist.html?" + encode_qs(state);
+        if (replace) {
+            window.history.replaceState(state, "", url);
+        } else {
+            window.history.pushState(state, "", url);
+        }
+    }
+
+    function weapon_predicate(state, weapon_data) {
+        var weapon_type = state["weapon_type"];
+        var weapon_element = state["weapon_element"];
+        var final_only = state["weapon_final"];
+        var weapon_names = state["weapon_name_text"].split("|");
+
+        if (weapon_type == "Bow" || weapon_type == "Light Bowgun"
+        || weapon_type == "Heavy Bowgun") {
+            // we only support blademaster weapons for now
+            return false;
+        }
+
+        console.log("check " + weapon_data["name"]);
+
+        if (final_only && weapon_data["final"] != 1) {
+            return false;
+        }
+
+        if (weapon_type != "All" && weapon_type != weapon_data["wtype"]) {
+            return false;
+        }
+
+        if (weapon_element != "All"
+        &&  weapon_element != weapon_data["element"]
+        &&  weapon_element != weapon_data["element_2"]
+        &&  weapon_element != weapon_data["awaken"]) {
+            if (weapon_element != "None"
+            ||  weapon_data["element"] != null
+            || weapon_data["awaken"] != null) {
+                return false;
+            }
+        }
+
+        if (weapon_names && !list_match(weapon_names, [weapon_data["name"]])) {
+            return false;
+        }
+
+        return true;
+    }
+
+    function list_match(needles, string_list) {
+        var found = false;
+        for (var i=0; i<string_list.length; i++) {
+            for (var j=0; j<needles.length; j++) {
+                if (string_list[i].search(needles[j]) >= 0) {
+                    found = true;
+                    break;
+                }
+                if (found) {
+                    break;
+                }
+            }
+        }
+        return found;
+    }
+
+    function update_weapon_list(state) {
+        var match_count = 0;
+        console.log("updating weapon list: " + JSON.stringify(state));
+        $("#weapon_table").empty();
+        var results = [];
+        $.each(WEAPON_LIST, function(i, weapon_data) {
+            if (weapon_predicate(state, weapon_data)) {
+                weapon_data["id"] = i;
+                weapon_data["awaken"] = null;
+                weapon_data["element_2"] = "";
+                weapon_data["element_2_attack"] = "";
+                weapon_data["phial"] = "";
+                weapon_data["shelling_type"] = "";
+                weapon_data["horn_notes"] = "";
+                match_count += 1;
+                set_sharpness_titles(weapon_data);
+                set_horn_melodies_title(weapon_data);
+                weapon_data["wtype_short"] =
+                          WEAPON_TYPE_ABBR[weapon_data["wtype"]];
+                weapon_data["ELEMENT_ABBR"] = ELEMENT_ABBR;
+                var html = template_row.render(weapon_data);
+                results.push([weapon_data, html]);
+            }
+        });
+        results.sort(function (a, b) {
+            avals = get_sort_values(a[0]);
+            bvals = get_sort_values(b[0]);
+            if (avals > bvals) {
+                return -1;
+            }
+            if (avals < bvals) {
+                return 1;
+            }
+            return 0;
+        });
+        $.each(results, function(i, pair) {
+            $("#weapon_table").append(pair[1]);
+        });
+        console.log("match count: " + match_count);
+
+    }
+
+    function get_sort_values(weapon_data) {
+        var sharp_reverse = Array.prototype.slice.call(
+                                        weapon_data["sharpness"]);
+        sharp_reverse.reverse();
+        return [
+            weapon_data["attack"],
+            sharp_reverse,
+            weapon_data["element_attack"],
+            weapon_data["affinity"],
+            weapon_data["num_slots"],
+            weapon_data["defense"]
+        ];
+    }
+  </script>
+</head>
+<body>
+<div>
+  <table>
+  <tr>
+    <td><label for="weapon_type"
+               title="Only show weapons of this type"
+        >Type:</label></td>
+    <td><select id="weapon_type">
+        <option value="All">All</option>
+        <option value="Great Sword">Great Sword</option>
+        <option value="Long Sword">Long Sword</option>
+        <option value="Sword and Shield">Sword and Shield</option>
+        <option value="Dual Blades">Dual Blades</option>
+        <option value="Hammer">Hammer</option>
+        <option value="Hunting Horn">Hunting Horn</option>
+        <option value="Lance">Lance</option>
+        <option value="Gunlance">Gunlance</option>
+        <option value="Switch Axe">Switch Axe</option>
+        <option value="Charge Blade">Charge Blade</option>
+        <option value="Insect Glaive">Insect Glaive</option>
+        <option value="Light Bowgun">Light Bowgun</option>
+        <option value="Heavy Bowgun">Heavy Bowgun</option>
+        <option value="Bow">Bow</option>
+    </select></td>
+    <td><label for="weapon_element"
+               title="Only show weapons with this element (native or requiring awaken)"
+        >Element:</label></td>
+    <td><select id="weapon_element">
+        <option value="All">All</option>
+        <option value="None">None</option>
+        <option value="Fire">Fire</option>
+        <option value="Water">Water</option>
+        <option value="Thunder">Thunder</option>
+        <option value="Ice">Ice</option>
+        <option value="Dragon">Dragon</option>
+        <option value="Poison">Poison</option>
+        <option value="Paralysis">Paralysis</option>
+        <option value="Sleep">Sleep</option>
+        <option value="Blast">Blast</option>
+    </select></td>
+    <td><label for="weapon_final"
+               title="Only show weapons with no furthur upgrades"
+        >Final?</label></td>
+    <td><input id="weapon_final" type="checkbox" /></td>
+    <td><button id="search">Search</button></td>
+  </tr>
+  <tr>
+    <td colspan="7">
+      <label for="weapon_name_text"
+             title="Show only weapons with a match in the name. List of strings separated by '|' (vertical bar)."
+             >Name:</label>
+      <input id="weapon_name_text" size="15" />
+    </td>
+  </tr>
+  </table>
+</div>
+<table id="weapon_table">
+</table>
+<div id="sharpness_popup"></div>
+</body>

web/templates/weaponrow.ejs

@@ -6,8 +6,12 @@
     <% } %>
 </td>
 <td>
+<% if (url) { %>
+  <a href="<%= url %>"><%= name %></a>
+<% } else { %>
   <a href="weaponplanner.html?weapon=<%= encodeURIComponent(name) %>"
         ><%= name %></a>
+<% } %>
 </td>
 <td><%= wtype_short %></td>
 <td style="text-align:right"><%= attack %></td>
@@ -15,15 +19,15 @@
 <td>
 <% if (awaken) { %>
   (<img style="height:.8em" title="Awaken <%= awaken %>"
-            src="img/<%= awaken %>.png" /><%= awaken_attack %>
+            src="/img/<%= awaken %>.png" /><%= awaken_attack %>
             <%= ELEMENT_ABBR[awaken] %>)
 <% } else if (element) { %>
   <img style="height:.8em" title="<%= element %>"
-            src="img/<%= element %>.png" /> <%= element_attack %>
+            src="/img/<%= element %>.png" /> <%= element_attack %>
             <%= ELEMENT_ABBR[element] %>
   <% if (element_2) { %>
     <img style="height:.8em" title="<%= element_2 %>"
-            src="img/<%= element_2 %>.png" /> <%= element_2_attack %>
+            src="/img/<%= element_2 %>.png" /> <%= element_2_attack %>
             <%= ELEMENT_ABBR[element_2] %>
   <% } %>
 <% } %>
@@ -41,7 +45,9 @@
 <span style="width:<%= sharpness[3] %>px" class="green"></span>
 <span style="width:<%= sharpness[4] %>px" class="blue"></span>
 <span style="width:<%= sharpness[5] %>px" class="white"></span>
+<% if (sharpness.length > 6) { %>
 <span style="width:<%= sharpness[6] %>px" class="purple"></span>
+<% } %>
 </div>
 <div class="sharpness-bar" title="<%= sharpness_all_title %>">
 <span style="width:<%= sharpness_plus[0] %>px" class="red"></span>
@@ -50,7 +56,9 @@
 <span style="width:<%= sharpness_plus[3] %>px" class="green"></span>
 <span style="width:<%= sharpness_plus[4] %>px" class="blue"></span>
 <span style="width:<%= sharpness_plus[5] %>px" class="white"></span>
+<% if (sharpness.length > 6) { %>
 <span style="width:<%= sharpness_plus[6] %>px" class="purple"></span>
+<% } %>
 </div>
 <% } %>
 </td>

web/weaponlist.html

@@ -246,6 +246,7 @@
                               data["wtype_short"] =
                                         WEAPON_TYPE_ABBR[data["wtype"]];
                               data["ELEMENT_ABBR"] = ELEMENT_ABBR;
+                              data["url"] = null;
                               var html = template_row.render(data);
                               $("#weapon_table").append(html);
                               var setup = get_calculating_palico_setup(data);