Add a wizard to Pcbnew to download 3d shapes libraries from Github

pcbnew/github/CMakeLists.txt

@@ -51,7 +51,7 @@ set( CMAKE_CXX_FLAGS
     "${CMAKE_CXX_FLAGS} -Wno-sign-compare -Wno-reorder -Wno-unused-variable -Wno-unused-function -Wno-strict-aliasing" )
 
 set( GITHUB_PLUGIN_SRCS
-    github_plugin.cpp github_getliblist.cpp
+    github_plugin.cpp github_getliblist.cpp html_link_parser.cpp
     )
 
 add_library( github_plugin STATIC ${GITHUB_PLUGIN_SRCS} )

pcbnew/github/github_getliblist.cpp

@@ -75,19 +75,55 @@
 #include <github_getliblist.h>
 #include <macros.h>
 #include <common.h>
+#include <html_link_parser.h>
 
 
 GITHUB_GETLIBLIST::GITHUB_GETLIBLIST( const wxString& aRepoURL )
 {
     m_repoURL = aRepoURL;
+    m_libs_ext =  wxT( ".pretty" );
+    strcpy( m_option_string, "application/json" );
 }
 
 
-bool GITHUB_GETLIBLIST::GetLibraryList( wxArrayString& aList )
+bool GITHUB_GETLIBLIST::Get3DshapesLibsList( wxArrayString* aList,
+                        bool (*aFilter)( const wxString& aData ) )
+{
+    std::string fullURLCommand;
+    strcpy( m_option_string, "text/html" );
+
+    wxString repoURL = m_repoURL;
+
+    wxString errorMsg;
+
+    fullURLCommand = repoURL.utf8_str();
+    bool success = remote_get_json( &fullURLCommand, &errorMsg );
+
+    if( !success )
+    {
+        wxMessageBox( errorMsg );
+        return false;
+    }
+
+    if( aFilter && aList )
+    {
+        //Convert m_image (std::string) to a wxString for HTML_LINK_PARSER
+        wxString buffer( GetBuffer() );
+
+        HTML_LINK_PARSER html_parser( buffer, *aList );
+        html_parser.ParseLinks( aFilter );
+    }
+
+    return true;
+}
+
+
+bool GITHUB_GETLIBLIST::GetFootprintLibraryList( wxArrayString& aList )
 {
     std::string fullURLCommand;
     int page = 1;
     int itemCountMax = 99;              // Do not use a valu > 100, it does not work
+    strcpy( m_option_string, "application/json" );
 
     // Github max items returned is 100 per page
 
@@ -101,12 +137,13 @@ bool GITHUB_GETLIBLIST::GetLibraryList( wxArrayString& aList )
     // The URL lib names are relative to the server name.
     // so add the server name to them.
     wxURI repo( m_repoURL );
-    wxString urlPrefix = wxT( "https://" ) + repo.GetServer() + wxT( "/" );
+    wxString urlPrefix = repo.GetScheme() + wxT( "://" ) + repo.GetServer() + wxT( "/" );
 
     wxString errorMsg;
     const char  sep = ',';      // Separator fields, in json returned file
     wxString    tmp;
     int items_count_per_page = 0;
+    std::string& json_image = GetBuffer();
 
     while( 1 )
     {
@@ -118,9 +155,10 @@ bool GITHUB_GETLIBLIST::GetLibraryList( wxArrayString& aList )
             return false;
         }
 
-        for( unsigned ii = 0; ii < m_json_image.size(); ii++ )
+
+        for( unsigned ii = 0; ii < json_image.size(); ii++ )
         {
-            if( m_json_image[ii] == sep || ii == m_json_image.size() - 1 )
+            if( json_image[ii] == sep || ii == json_image.size() - 1 )
             {
                 if( tmp.StartsWith( wxT( "\"full_name\"" ) ) )
                 {
@@ -129,7 +167,7 @@ bool GITHUB_GETLIBLIST::GetLibraryList( wxArrayString& aList )
                     if( tmp[tmp.Length() - 1] == QUOTE )
                         tmp.RemoveLast();
 
-                    if( tmp.EndsWith( wxT( ".pretty" ) ) )
+                    if( tmp.EndsWith( m_libs_ext ) )
                     {
                         aList.Add( tmp.AfterLast( ':' ) );
                         int idx = aList.GetCount() - 1;
@@ -146,7 +184,7 @@ bool GITHUB_GETLIBLIST::GetLibraryList( wxArrayString& aList )
                 tmp.Clear();
             }
             else
-                tmp << m_json_image[ii];
+                tmp << json_image[ii];
         }
 
         if( items_count_per_page >= itemCountMax )
@@ -154,7 +192,7 @@ bool GITHUB_GETLIBLIST::GetLibraryList( wxArrayString& aList )
             page++;
             repoURL2listURL( m_repoURL, &fullURLCommand, itemCountMax, page );
             items_count_per_page = 0;
-            m_json_image.clear();
+            ClearBuffer();
         }
         else
             break;

pcbnew/github/github_getliblist.h

@@ -27,19 +27,58 @@
 
 
 /**
- *   Class GITHUB_GETLIBLIST
- *   implements a portion of pcbnew's PLUGIN interface to provide read only access
- *   to a github repo to extract pretty footprints library list, in json format.
+ * Class GITHUB_GETLIBLIST
+ * implements a portion of pcbnew's PLUGIN interface to provide read only access
+ * to a github repo to extract pretty footprints library list, in json format,
+ * or extract 3D shapes libraries list (.3dshapes folders) and download the 3d shapes files
  *
- *   this plugin simply reads in a zip file of the repo and unzips it from RAM as
- *   needed. Therefore this "Github" plugin is <b>read only for accessing remote
- *   at https://api.github.com/orgs/KiCad/repos</b>
+ * To extract pretty footprints library list, this plugin simply reads in
+ * a zip file of the repo and unzips it from RAM as needed.
+ * Therefore this "Github" plugin is <b>read only for accessing remote
+ * at https://api.github.com/orgs/KiCad/repos</b>
+ *
+ * To extract 3D shapes libraries list (.3dshapes folders) we cannot use api.github.com
+ * to read this list, becuse it is in a subdirectory of https://github.com/KiCad.
+ * The corresponding html page of the server is read, and URLs of all .3dshapes folders
+ * are extracted.
+ * files are then read from https://raw.githubusercontent.com/<lib path>, but not zipped
+ * because they are not accessible in zipped form.
  */
 class GITHUB_GETLIBLIST
 {
 public:
     // -----<API>----------------------------------------------------------
-    bool GetLibraryList( wxArrayString& aList );
+
+    /**
+     * Fills aList by the name of footprint libraries found on the github repo
+     */
+    bool GetFootprintLibraryList( wxArrayString& aList );
+
+    /**
+     * Fills aList by the URL of libraries found on the github repo
+     * @param aList = a reference to a wxArrayString to fill with names
+     * @param aFilter( const wxString& aData ) = a callback funtion to
+     * to filter URLs to put in aList.
+     * If NULL, no URL will be stored in aList
+     */
+    bool Get3DshapesLibsList( wxArrayString* aList,
+                        bool (*aFilter)( const wxString& aData ) );
+
+    /**
+     * @return the buffer which stores all the downloaded raw data
+     */
+    std::string& GetBuffer() { return m_image; }
+
+    /**
+     * Clear the buffer which stores all the downloaded raw data
+     */
+    void ClearBuffer() { m_image.clear(); }
+
+    /**
+     * The library names are expecting ending by .pretty
+     * SetLibraryExt set the extension to aExt
+     */
+    void SetLibraryExt( const wxString& aExt ) { m_libs_ext = aExt; }
 
     // -----</API>---------------------------------------------------------
 
@@ -55,7 +94,7 @@ protected:
      *
      * @param  aRepoURL points to the base of the repo.
      * @param  aFullURLCommand is URL the full URL command (URL+options).
-     * @param  aItemCountMax is the max item count in apage,
+     * @param  aItemCountMax is the max item count in a page,
      *   and is 100 for github repo.
      * @param  aPage is the page number, if there are more than one page in repo.
      * @return bool - true if @a aRepoULR was parseable, else false
@@ -75,8 +114,10 @@ protected:
     bool remote_get_json( std::string* aFullURLCommand, wxString* aMsgError );
 
     wxString m_github_path;     ///< Something like https://api.github.com/orgs/KiCad
-    std::string m_json_image;   ///< image of the text file in its entirety.
-    wxString m_repoURL;         // the URL of the Github repo
+    std::string m_image;        ///< image of the downloaded data in its entirety.
+    wxString m_repoURL;         ///< the URL of the Github repo
+    wxString m_libs_ext;        ///< the extension of the name of libraries (default = .pretty)
+    char m_option_string[64];   ///< option for transfert type, like "application/json"
 };
 
 

pcbnew/github/github_plugin.cpp

@@ -592,8 +592,9 @@ bool GITHUB_GETLIBLIST::remote_get_json( std::string* aFullURLCommand, wxString*
     avhttp::http_stream     h( io );
     avhttp::request_opts    options;
 
-    options.insert( "Accept",       "application/json" );
-    options.insert( "User-Agent",   "http://kicad-pcb.org" );   // THAT WOULD BE ME.
+
+    options.insert( "Accept", m_option_string );
+    options.insert( "User-Agent", "http://kicad-pcb.org" );   // THAT WOULD BE ME.
     h.request_options( options );
 
     try
@@ -603,8 +604,8 @@ bool GITHUB_GETLIBLIST::remote_get_json( std::string* aFullURLCommand, wxString*
         h.open( *aFullURLCommand );      // only one file, therefore do it synchronously.
         os << &h;
 
-        // Keep json text file image in RAM.
-        m_json_image = os.str();
+        // Keep downloaded text file image in RAM.
+        m_image = os.str();
 
         // 4 lines, using SSL, top that.
     }
@@ -613,7 +614,7 @@ bool GITHUB_GETLIBLIST::remote_get_json( std::string* aFullURLCommand, wxString*
         // https "GET" has faild, report this to API caller.
         static const char errorcmd[] = "https GET command failed";  // Do not translate this message
 
-        UTF8 fmt( _( "%s\nCannot get/download json data from: '%s'\nReason: '%s'" ) );
+        UTF8 fmt( _( "%s\nCannot get/download data from: '%s'\nReason: '%s'" ) );
 
         std::string msg = StrPrintf( fmt.c_str(),
                 errorcmd,
@@ -625,7 +626,7 @@ bool GITHUB_GETLIBLIST::remote_get_json( std::string* aFullURLCommand, wxString*
 
         if( aMsgError )
         {
-            *aMsgError =  msg;
+            *aMsgError = FROM_UTF8( msg.c_str() );
             return false;
         }
     }

pcbnew/github/html_link_parser.cpp

@@ -0,0 +1,68 @@
+/**
+ * @file html_link_parse.cpp
+ */
+
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 1992-2014 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you may find one here:
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+ * or you may search the http://www.gnu.org website for the version 2 license,
+ * or you may write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+ */
+
+/*
+ * wxWidgets gives very few info about wxwebkit. For more info and more comments:
+ * see https://forums.wxwidgets.org/viewtopic.php?f=1&t=1119#
+ */
+
+#include <html_link_parser.h>
+
+bool LINK_TAGHANDLER::HandleTag(const wxHtmlTag& tag)
+{
+    if( tag.HasParam( wxT("HREF") ) )
+    {
+        wxString href( tag.GetParam( wxT("HREF") ) );
+        // Add the first parameter (the link)
+        m_Parser->AddString( href );
+        // Parse other params, but do nothing, because the AddText() callback
+        // do nothing
+        ParseInner(tag);
+        return true;
+    }
+    else
+        return false;
+}
+
+HTML_LINK_PARSER::HTML_LINK_PARSER( const wxString& aSrc, wxArrayString& aUrls )
+        : m_src( aSrc ), stringUrls( aUrls )
+{
+    AddTagHandler( new LINK_TAGHANDLER(this) );
+}
+
+
+void HTML_LINK_PARSER::AddString( const wxString& aText )
+{
+    wxString text = aText;
+    text.Trim( true );
+    text.Trim( false );
+
+    if( ! m_filter || m_filter( text ) )
+    {
+        stringUrls.Add( text );
+    }
+}

pcbnew/github/html_link_parser.h

@@ -0,0 +1,109 @@
+/**
+ * @file html_link_parse.h
+ */
+
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 1992-2014 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you may find one here:
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+ * or you may search the http://www.gnu.org website for the version 2 license,
+ * or you may write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+ */
+
+/*
+ * wxWidgets gives very few info about wxwebkit. For more info and more comments:
+ * see https://forums.wxwidgets.org/viewtopic.php?f=1&t=1119#
+ */
+
+#ifndef HTML_LINK_PARSE_H
+#define HTML_LINK_PARSE_H
+
+
+#include <wx/wx.h>
+#include <wx/html/htmlpars.h>
+
+class HTML_LINK_PARSER ;
+
+/**
+ * a Tag parser, to extract tagged data in html text.
+ * this tag handler extract a url link, givent by tag "A"
+ * like:
+ * "<a href="/KiCad/Valves.pretty" itemprop="name codeRepository"> Valves.pretty</a>"
+ * the tag is "a"
+ * and the link is the parameter given by "href"
+ */
+class LINK_TAGHANDLER : public wxHtmlTagHandler
+{
+    HTML_LINK_PARSER* m_Parser;
+
+public:
+    LINK_TAGHANDLER() : m_Parser( NULL )
+    {
+    }
+
+    LINK_TAGHANDLER( HTML_LINK_PARSER* aParser ) : m_Parser( aParser )
+    {
+    }
+
+    wxString GetSupportedTags()
+    {
+        return "A";
+    }
+
+    bool HandleTag(const wxHtmlTag& tag);
+};
+
+/**
+ * The engine to parse a html text and extract useful data
+ * Here, the useful data are url links
+ */
+class HTML_LINK_PARSER : public wxHtmlParser
+{
+    const wxString& m_src;          // the html text to parse
+    wxArrayString& stringUrls;      // the strings extracted from html text
+    bool (*m_filter)( const wxString& aData );  // a callback function to filter strings
+
+public:
+
+    HTML_LINK_PARSER( const wxString& aSrc, wxArrayString& aUrls );
+
+    /**
+     * Parse the html text and store links in stringUrls
+     * Stored links can be filtered if aFilter is non NULL
+     * @param aFilter a filtering function ( bool aFilter( const wxString& aData ) )
+     * which return true if the text aData must be stored.
+     */
+    void ParseLinks( bool (*aFilter)( const wxString& aData ) )
+    {
+        m_filter = aFilter;
+        Parse(m_src);
+    }
+
+    // virtual pure from wxHtmlParser, do nothing here, but needed.
+    void AddText( const wxString& aText ){}
+
+    // Our "AddText" used to store selected text (the url link)
+    void AddString( const wxString& aText );
+
+    wxObject* GetProduct()
+    {
+        return NULL;
+    }
+};
+
+#endif      // ifndef HTML_LINK_PARSE_H