electron-rare/FreeCAD
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

App: add string table

Browse Source 
General purpose string table to avoid saving duplicated text string
multiple times.. Mainly used by topological naming to compress model
histroy. But can be used for other purpose as well, e.g.  to strore
documentation of dynamic property.
This commit is contained in:
Zheng, Lei
2022-09-03 00:36:31 +05:00
committed by Uwe
parent fd60f29965
commit 444de2c8af
12 changed files with 3087 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/App/Application.cpp
+5
View File
@@ -102,6 +102,7 @@
#include "OriginFeature.h"
#include "OriginGroupExtension.h"
#include "OriginGroupExtensionPy.h"
#include "StringHasherPy.h"
#include "Part.h"
#include "PartPy.h"
#include "Placement.h"
@@ -277,6 +278,7 @@ void Application::setupPythonTypes()
Base::Interpreter().addType(&Base::PlacementPy::Type, pAppModule, "Placement");
Base::Interpreter().addType(&Base::RotationPy::Type, pAppModule, "Rotation");
Base::Interpreter().addType(&Base::AxisPy::Type, pAppModule, "Axis");
Base::Interpreter().addType(&App::StringHasherPy::Type, pAppModule, "StringHasher");
// Note: Create an own module 'Base' which should provide the python
// binding classes from the base module. At a later stage we should
@@ -1904,6 +1906,9 @@ void Application::initTypes()
Data::ComplexGeoData ::init();
Data::Segment ::init();
App::StringID ::init();
App::StringHasher ::init();
// Properties
App ::Property ::init();
App ::PropertyContainer ::init();
src/App/CMakeLists.txt
+10
View File
@@ -87,6 +87,8 @@ else()
)
endif()
generate_from_xml(StringIDPy)
generate_from_xml(StringHasherPy)
generate_from_xml(DocumentPy)
generate_from_xml(DocumentObjectPy)
generate_from_xml(ExtensionPy)
@@ -109,6 +111,8 @@ generate_from_py(FreeCADInit InitScript.h)
generate_from_py(FreeCADTest TestScript.h)
SET(FreeCADApp_XML_SRCS
StringIDPy.xml
StringHasherPy.xml
ExtensionPy.xml
ExtensionContainerPy.xml
DocumentObjectExtensionPy.xml
@@ -131,7 +135,11 @@ SOURCE_GROUP("XML" FILES ${FreeCADApp_XML_SRCS})
# The document stuff
SET(Document_CPP_SRCS
Annotation.cpp
StringHasher.cpp
StringHasherPyImp.cpp
StringIDPyImp.cpp
Document.cpp
DocumentParams.cpp
DocumentObject.cpp
Extension.cpp
ExtensionPyImp.cpp
@@ -178,7 +186,9 @@ SET(Document_CPP_SRCS
SET(Document_HPP_SRCS
Annotation.h
StringHasher.h
Document.h
DocumentParams.h
DocumentObject.h
Extension.h
ExtensionContainer.h
src/App/Document.cpp
+97 -16
View File
@@ -65,6 +65,7 @@ recompute path. Also, it enables more complicated dependencies beyond trees.
#include <boost/algorithm/string.hpp>
#include <boost/graph/graphviz.hpp>
#include <boost/bimap.hpp>
#include <boost/graph/strong_components.hpp>
#ifdef USE_OLD_DAG
@@ -107,6 +108,7 @@ recompute path. Also, it enables more complicated dependencies beyond trees.
#include "MergeDocuments.h"
#include "Origin.h"
#include "OriginGroupExtension.h"
#include "StringHasher.h"
#include "Transactions.h"
#ifdef _MSC_VER
@@ -117,7 +119,6 @@ recompute path. Also, it enables more complicated dependencies beyond trees.
#include <zipios++/zipoutputstream.h>
#include <zipios++/meta-iostreams.h>
FC_LOG_LEVEL_INIT("App", true, true, true)
using Base::Console;
@@ -152,8 +153,10 @@ typedef std::vector <size_t> Path;
namespace App {
typedef boost::bimap<StringHasherRef,int> HasherMap;
static bool _IsRestoring;
static bool _IsRelabeling;
// Pimpl class
struct DocumentP
{
@@ -179,6 +182,7 @@ struct DocumentP
unsigned int UndoMemSize;
unsigned int UndoMaxStackSize;
std::string programVersion;
mutable HasherMap hashers;
#ifdef USE_OLD_DAG
DependencyList DepList;
std::map<DocumentObject*,Vertex> VertexObjectList;
@@ -187,10 +191,15 @@ struct DocumentP
std::multimap<const App::DocumentObject*,
std::unique_ptr<App::DocumentObjectExecReturn> > _RecomputeLog;
StringHasherRef Hasher;
// restored files
std::set<std::string> files;
DocumentP() {
#ifdef FC_NO_RANDOM_ID
lastObjectId = 10;
#else
static std::random_device _RD;
static std::mt19937 _RGEN(_RD());
static std::uniform_int_distribution<> _RDIST(0,5000);
@@ -198,6 +207,8 @@ struct DocumentP
// copying shape from other document. It is probably better to randomize
// on each object ID.
lastObjectId = _RDIST(_RGEN);
#endif
Hasher.reset(new StringHasher);
activeObject = nullptr;
activeUndoTransaction = nullptr;
iTransactionMode = 0;
@@ -1470,7 +1481,6 @@ void Document::onChanged(const Property* prop)
// the Name property is a label for display purposes
if (prop == &Label) {
Base::FlagToggler<> flag(_IsRelabeling);
App::GetApplication().signalRelabelDocument(*this);
} else if(prop == &ShowHidden) {
App::GetApplication().signalShowHidden(*this);
@@ -1505,6 +1515,12 @@ void Document::onChanged(const Property* prop)
// recursive call of onChanged()
this->Uid.setValue(id);
}
} else if(prop == &UseHasher) {
for(auto obj : d->objectArray) {
auto geofeature = dynamic_cast<GeoFeature*>(obj);
if(geofeature && geofeature->getPropertyOfGeometry())
geofeature->enforceRecompute();
}
}
}
@@ -1512,8 +1528,8 @@ void Document::onBeforeChangeProperty(const TransactionalObject *Who, const Prop
{
if(Who->isDerivedFrom(App::DocumentObject::getClassTypeId()))
signalBeforeChangeObject(*static_cast<const App::DocumentObject*>(Who), *What);
if(!d->rollback && !_IsRelabeling) {
_checkTransaction(nullptr,What,__LINE__);
if(!d->rollback) {
_checkTransaction(nullptr, What, __LINE__);
if (d->activeUndoTransaction)
d->activeUndoTransaction->addObjectChange(Who,What);
}
@@ -1548,10 +1564,8 @@ Document::Document(const char *name)
Console().Log("+App::Document: %p\n",this);
#endif
std::string CreationDateString = Base::TimeInfo::currentDateTimeString();
std::string Author = App::GetApplication().GetParameterGroupByPath
("User parameter:BaseApp/Preferences/Document")->GetASCII("prefAuthor","");
std::string AuthorComp = App::GetApplication().GetParameterGroupByPath
("User parameter:BaseApp/Preferences/Document")->GetASCII("prefCompany","");
std::string Author = DocumentParams::getprefAuthor();
std::string AuthorComp = DocumentParams::getprefCompany();
ADD_PROPERTY_TYPE(Label,("Unnamed"),0,Prop_None,"The name of the document");
ADD_PROPERTY_TYPE(FileName,(""),0,PropertyType(Prop_Transient|Prop_ReadOnly),"The path to the file where the document is saved to");
ADD_PROPERTY_TYPE(CreatedBy,(Author.c_str()),0,Prop_None,"The creator of the document");
@@ -1572,8 +1586,7 @@ Document::Document(const char *name)
ADD_PROPERTY_TYPE(LicenseURL,("http://creativecommons.org/licenses/by/3.0/"),0,Prop_None,"URL to the license text/contract");
// license stuff
int licenseId = App::GetApplication().GetParameterGroupByPath
("User parameter:BaseApp/Preferences/Document")->GetInt("prefLicenseType",0);
int licenseId = DocumentParams::getprefLicenseType();
std::string license;
std::string licenseUrl;
switch (licenseId) {
@@ -1618,13 +1631,17 @@ Document::Document(const char *name)
break;
}
licenseUrl = App::GetApplication().GetParameterGroupByPath
("User parameter:BaseApp/Preferences/Document")->GetASCII("prefLicenseUrl", licenseUrl.c_str());
if(DocumentParams::getprefLicenseUrl().empty())
licenseUrl = DocumentParams::getprefLicenseUrl();
ADD_PROPERTY_TYPE(License,(license.c_str()),0,Prop_None,"License string of the Item");
ADD_PROPERTY_TYPE(LicenseURL,(licenseUrl.c_str()),0,Prop_None,"URL to the license text/contract");
ADD_PROPERTY_TYPE(ShowHidden,(false), 0,PropertyType(Prop_None),
"Whether to show hidden object items in the tree view");
ADD_PROPERTY_TYPE(UseHasher,(true), 0,PropertyType(Prop_Hidden),
"Whether to use hasher on topological naming");
if(!DocumentParams::getUseHasher())
UseHasher.setValue(false);
// this creates and sets 'TransientDir' in onChanged()
ADD_PROPERTY_TYPE(TransientDir,(""),0,PropertyType(Prop_Transient|Prop_ReadOnly),
@@ -1694,11 +1711,27 @@ std::string Document::getTransientDirectoryName(const std::string& uuid, const s
void Document::Save (Base::Writer &writer) const
{
d->hashers.clear();
addStringHasher(d->Hasher);
writer.Stream() << "<Document SchemaVersion=\"4\" ProgramVersion=\""
<< App::Application::Config()["BuildVersionMajor"] << "."
<< App::Application::Config()["BuildVersionMinor"] << "R"
<< App::Application::Config()["BuildRevision"]
<< "\" FileVersion=\"" << writer.getFileVersion() << "\">" << endl;
<< "\" FileVersion=\"" << writer.getFileVersion()
<< "\" Uid=\"" << Uid.getValueStr()
<< "\" StringHasher=\"1\">\n";
writer.incInd();
// NOTE: DO NOT save the main string hasher as separate file, because it is
// required by many objects, which assume the string hasher is fully
// restored.
d->Hasher->setPersistenceFileName(0);
d->Hasher->Save(writer);
writer.decInd();
PropertyContainer::Save(writer);
@@ -1710,7 +1743,12 @@ void Document::Save (Base::Writer &writer) const
void Document::Restore(Base::XMLReader &reader)
{
int i,Cnt;
d->hashers.clear();
d->touchedObjs.clear();
addStringHasher(d->Hasher);
Base::ReaderContext rctx(getName());
setStatus(Document::PartialDoc,false);
reader.readElement("Document");
@@ -1727,6 +1765,11 @@ void Document::Restore(Base::XMLReader &reader)
reader.FileVersion = 0;
}
if (reader.hasAttribute("StringHasher")) {
Base::ReaderContext rctx("StringHasher");
d->Hasher->Restore(reader);
} else
d->Hasher->clear();
// When this document was created the FileName and Label properties
// were set to the absolute path or file name, respectively. To save
// the document to the file it was loaded from or to show the file name
@@ -1769,6 +1812,7 @@ void Document::Restore(Base::XMLReader &reader)
for (i=0 ;i<Cnt ;i++) {
reader.readElement("Feature");
string name = reader.getAttribute("name");
Base::ReaderContext rctx(name);
DocumentObject* pObj = getObject(name.c_str());
if (pObj) { // check if this feature has been registered
pObj->setStatus(ObjectStatus::Restore, true);
@@ -1791,6 +1835,36 @@ void Document::Restore(Base::XMLReader &reader)
reader.readEndElement("Document");
}
std::pair<bool,int> Document::addStringHasher(const StringHasherRef & hasher) const {
if (!hasher)
return std::make_pair(false, 0);
auto ret = d->hashers.left.insert(HasherMap::left_map::value_type(hasher,(int)d->hashers.size()));
if (ret.second)
hasher->clearMarks();
return std::make_pair(ret.second,ret.first->second);
}
StringHasherRef Document::getHasher() const {
return d->Hasher;
}
StringHasherRef Document::getStringHasher(int idx) const {
StringHasherRef hasher;
if(idx<0) {
if(UseHasher.getValue())
return d->Hasher;
return hasher;
}
auto it = d->hashers.right.find(idx);
if(it == d->hashers.right.end()) {
hasher = new StringHasher;
d->hashers.right.insert(HasherMap::right_map::value_type(idx,hasher));
}else
hasher = it->second;
return hasher;
}
struct DocExportStatus {
Document::ExportStatus status;
std::set<const App::DocumentObject*> objs;
@@ -1827,6 +1901,7 @@ Document::ExportStatus Document::isExporting(const App::DocumentObject *obj) con
void Document::exportObjects(const std::vector<App::DocumentObject*>& obj, std::ostream& out) {
DocumentExporting exporting(obj);
d->hashers.clear();
if(FC_LOG_INSTANCE.isEnabled(FC_LOGLEVEL_LOG)) {
for(auto o : obj) {
@@ -1863,6 +1938,7 @@ void Document::exportObjects(const std::vector<App::DocumentObject*>& obj, std::
// write additional files
writer.writeFiles();
d->hashers.clear();
}
#define FC_ATTR_DEPENDENCIES "Dependencies"
@@ -2166,14 +2242,16 @@ Document::readObjects(Base::XMLReader& reader)
void Document::addRecomputeObject(DocumentObject *obj) {
if(testStatus(Status::Restoring) && obj) {
setStatus(Status::RecomputeOnRestore, true);
d->touchedObjs.insert(obj);
obj->touch();
obj->enforceRecompute();
}
}
std::vector<App::DocumentObject*>
Document::importObjects(Base::XMLReader& reader)
{
d->hashers.clear();
Base::FlagToggler<> flag(_IsRestoring,false);
Base::ObjectStatusLocker<Status, Document> restoreBit(Status::Restoring, this);
Base::ObjectStatusLocker<Status, Document> restoreBit2(Status::Importing, this);
@@ -2192,6 +2270,7 @@ Document::importObjects(Base::XMLReader& reader)
reader.FileVersion = 0;
}
Base::ReaderContext rctx(getName());
std::vector<App::DocumentObject*> objs = readObjects(reader);
for(auto o : objs) {
if(o && o->getNameInDocument()) {
@@ -2224,7 +2303,7 @@ Document::importObjects(Base::XMLReader& reader)
if(o && o->getNameInDocument())
o->setStatus(App::ObjImporting,false);
}
d->hashers.clear();
return objs;
}
@@ -2237,6 +2316,8 @@ unsigned int Document::getMemSize (void) const
for (it = d->objectArray.begin(); it != d->objectArray.end(); ++it)
size += (*it)->getMemSize();
size += d->Hasher->getMemSize();
// size of the document properties...
size += PropertyContainer::getMemSize();
src/App/Document.h
+46
View File
@@ -23,13 +23,22 @@
#ifndef APP_DOCUMENT_H
#define APP_DOCUMENT_H
#include <CXX/Objects.hxx>
#include <Base/Observer.h>
#include <Base/Persistence.h>
#include <Base/Type.h>
#include <Base/Handle.h>
#include "PropertyContainer.h"
#include "PropertyLinks.h"
#include "PropertyStandard.h"
#include <memory>
#include <map>
#include <vector>
class QByteArray;
namespace Base {
class Writer;
}
@@ -43,6 +52,8 @@ namespace App
class DocumentPy; // the python document class
class Application;
class Transaction;
class StringHasher;
typedef Base::Reference<StringHasher> StringHasherRef;
}
namespace App
@@ -68,6 +79,7 @@ public:
RestoreError = 10,
LinkStampChanged = 11, // Indicates during restore time if any linked document's time stamp has changed
IgnoreErrorOnRecompute = 12, // Don't report errors if the recompute failed
RecomputeOnRestore = 13, // Mark pending recompute on restore for migration purpose
};
/** @name Properties */
@@ -109,6 +121,8 @@ public:
PropertyString TipName;
/// Whether to show hidden items in TreeView
PropertyBool ShowHidden;
/// Whether to use hasher on topological naming
PropertyBool UseHasher;
//@}
/** @name Signals of the document */
@@ -473,6 +487,38 @@ public:
(const App::DocumentObject* from, const App::DocumentObject* to) const;
//@}
/** Called by property during properly save its containing StringHasher
*
* @param hasher: the input hasher
* @return Returns a pair<bool,int>. Boolean member indicate if the
* StringHasher has been saved before. The Integer is the hasher index.
*
* The StringHasher object is designed to be shared among multiple objects.
* So, we must not save duplicate copies of the same hasher. And must be
* able to restore with the same sharing relationship. This function returns
* whether the hasher has been saved before by other objects, and the index
* of the hasher. If the hasher has not been saved before, the object must
* save the hasher by calling StringHasher::Save
*/
std::pair<bool,int> addStringHasher(const StringHasherRef & hasher) const;
/** Called by property to restore its containing StringHasher
*
* @param index: the index previously returned by calling addStringHasher()
* during save. Or if is negative, then return document's own string hasher
* if UseHasher is True
*
* @return Return the resulting string hasher.
*
* The caller is responsible to restore the hasher itself if it is the first
* owner of the hasher, i.e. return addStringHasher() returns true during
* save
*/
StringHasherRef getStringHasher(int index=-1) const;
/// Return the document's own hasher regardless of UseHasher
StringHasherRef getHasher() const;
/** Return the links to a given object
*
* @param links: holds the links found
src/App/DocumentParams.cpp
+1081
View File
File diff suppressed because it is too large Load Diff
src/App/DocumentParams.h
+359
View File
@@ -0,0 +1,359 @@
/****************************************************************************
* Copyright (c) 2022 Zheng Lei (realthunder) <[email protected]> *
* *
* This file is part of the FreeCAD CAx development system. *
* *
* This library is free software; you can redistribute it and/or *
* modify it under the terms of the GNU Library General Public *
* License as published by the Free Software Foundation; either *
* version 2 of the License, or (at your option) any later version. *
* *
* This library is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Library General Public License for more details. *
* *
* You should have received a copy of the GNU Library General Public *
* License along with this library; see the file COPYING.LIB. If not, *
* write to the Free Software Foundation, Inc., 59 Temple Place, *
* Suite 330, Boston, MA 02111-1307, USA *
* *
****************************************************************************/
#ifndef DOCUMENT_PARAMS_H
#define DOCUMENT_PARAMS_H
/*[[[cog
import DocumentParams
DocumentParams.declare()
]]]*/
// Auto generated code (Tools/params_utils.py:68)
#include <Base/Parameter.h>
#include <boost_signals2.hpp>
// Auto generated code (Tools/params_utils.py:74)
namespace App {
/** Convenient class to obtain App::Document related parameters
* The parameters are under group "User parameter:BaseApp/Preferences/Document"
*
* This class is auto generated by App/DocumentParams.py. Modify that file
* instead of this one, if you want to add any parameter. You need
* to install Cog Python package for code generation:
* @code
* pip install cogapp
* @endcode
*
* Once modified, you can regenerate the header and the source file,
* @code
* python3 -m cogapp -r App/DocumentParams.h App/DocumentParams.cpp
* @endcode
*
* You can add a new parameter by adding lines in App/DocumentParams.py. Available
* parameter types are 'Int, UInt, String, Bool, Float'. For example, to add
* a new Int type parameter,
* @code
* ParamInt(parameter_name, default_value, documentation, on_change=False)
* @endcode
*
* If there is special handling on parameter change, pass in on_change=True.
* And you need to provide a function implementation in App/DocumentParams.cpp with
* the following signature.
* @code
* void DocumentParams:on<parameter_name>Changed()
* @endcode
*/
class AppExport DocumentParams {
public:
static ParameterGrp::handle getHandle();
static boost::signals2::signal<void (const char*)> &signalParamChanged();
static void signalAll();
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter prefAuthor
static const std::string & getprefAuthor();
static const std::string & defaultprefAuthor();
static void removeprefAuthor();
static void setprefAuthor(const std::string &v);
static const char *docprefAuthor();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter prefSetAuthorOnSave
static const bool & getprefSetAuthorOnSave();
static const bool & defaultprefSetAuthorOnSave();
static void removeprefSetAuthorOnSave();
static void setprefSetAuthorOnSave(const bool &v);
static const char *docprefSetAuthorOnSave();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter prefCompany
static const std::string & getprefCompany();
static const std::string & defaultprefCompany();
static void removeprefCompany();
static void setprefCompany(const std::string &v);
static const char *docprefCompany();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter prefLicenseType
static const long & getprefLicenseType();
static const long & defaultprefLicenseType();
static void removeprefLicenseType();
static void setprefLicenseType(const long &v);
static const char *docprefLicenseType();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter prefLicenseUrl
static const std::string & getprefLicenseUrl();
static const std::string & defaultprefLicenseUrl();
static void removeprefLicenseUrl();
static void setprefLicenseUrl(const std::string &v);
static const char *docprefLicenseUrl();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter CompressionLevel
static const long & getCompressionLevel();
static const long & defaultCompressionLevel();
static void removeCompressionLevel();
static void setCompressionLevel(const long &v);
static const char *docCompressionLevel();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter CheckExtension
static const bool & getCheckExtension();
static const bool & defaultCheckExtension();
static void removeCheckExtension();
static void setCheckExtension(const bool &v);
static const char *docCheckExtension();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter ForceXML
static const long & getForceXML();
static const long & defaultForceXML();
static void removeForceXML();
static void setForceXML(const long &v);
static const char *docForceXML();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter SplitXML
static const bool & getSplitXML();
static const bool & defaultSplitXML();
static void removeSplitXML();
static void setSplitXML(const bool &v);
static const char *docSplitXML();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter PreferBinary
static const bool & getPreferBinary();
static const bool & defaultPreferBinary();
static void removePreferBinary();
static void setPreferBinary(const bool &v);
static const char *docPreferBinary();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter AutoRemoveFile
static const bool & getAutoRemoveFile();
static const bool & defaultAutoRemoveFile();
static void removeAutoRemoveFile();
static void setAutoRemoveFile(const bool &v);
static const char *docAutoRemoveFile();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter BackupPolicy
static const bool & getBackupPolicy();
static const bool & defaultBackupPolicy();
static void removeBackupPolicy();
static void setBackupPolicy(const bool &v);
static const char *docBackupPolicy();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter CreateBackupFiles
static const bool & getCreateBackupFiles();
static const bool & defaultCreateBackupFiles();
static void removeCreateBackupFiles();
static void setCreateBackupFiles(const bool &v);
static const char *docCreateBackupFiles();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter UseFCBakExtension
static const bool & getUseFCBakExtension();
static const bool & defaultUseFCBakExtension();
static void removeUseFCBakExtension();
static void setUseFCBakExtension(const bool &v);
static const char *docUseFCBakExtension();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter SaveBackupDateFormat
static const std::string & getSaveBackupDateFormat();
static const std::string & defaultSaveBackupDateFormat();
static void removeSaveBackupDateFormat();
static void setSaveBackupDateFormat(const std::string &v);
static const char *docSaveBackupDateFormat();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter CountBackupFiles
static const long & getCountBackupFiles();
static const long & defaultCountBackupFiles();
static void removeCountBackupFiles();
static void setCountBackupFiles(const long &v);
static const char *docCountBackupFiles();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter OptimizeRecompute
static const bool & getOptimizeRecompute();
static const bool & defaultOptimizeRecompute();
static void removeOptimizeRecompute();
static void setOptimizeRecompute(const bool &v);
static const char *docOptimizeRecompute();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter CanAbortRecompute
static const bool & getCanAbortRecompute();
static const bool & defaultCanAbortRecompute();
static void removeCanAbortRecompute();
static void setCanAbortRecompute(const bool &v);
static const char *docCanAbortRecompute();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter UseHasher
static const bool & getUseHasher();
static const bool & defaultUseHasher();
static void removeUseHasher();
static void setUseHasher(const bool &v);
static const char *docUseHasher();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter ViewObjectTransaction
static const bool & getViewObjectTransaction();
static const bool & defaultViewObjectTransaction();
static void removeViewObjectTransaction();
static void setViewObjectTransaction(const bool &v);
static const char *docViewObjectTransaction();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter WarnRecomputeOnRestore
static const bool & getWarnRecomputeOnRestore();
static const bool & defaultWarnRecomputeOnRestore();
static void removeWarnRecomputeOnRestore();
static void setWarnRecomputeOnRestore(const bool &v);
static const char *docWarnRecomputeOnRestore();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter NoPartialLoading
static const bool & getNoPartialLoading();
static const bool & defaultNoPartialLoading();
static void removeNoPartialLoading();
static void setNoPartialLoading(const bool &v);
static const char *docNoPartialLoading();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter ThumbnailNoBackground
static const bool & getThumbnailNoBackground();
static const bool & defaultThumbnailNoBackground();
static void removeThumbnailNoBackground();
static void setThumbnailNoBackground(const bool &v);
static const char *docThumbnailNoBackground();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter ThumbnailSampleSize
static const long & getThumbnailSampleSize();
static const long & defaultThumbnailSampleSize();
static void removeThumbnailSampleSize();
static void setThumbnailSampleSize(const long &v);
static const char *docThumbnailSampleSize();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter DuplicateLabels
static const bool & getDuplicateLabels();
static const bool & defaultDuplicateLabels();
static void removeDuplicateLabels();
static void setDuplicateLabels(const bool &v);
static const char *docDuplicateLabels();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter TransactionOnRecompute
static const bool & getTransactionOnRecompute();
static const bool & defaultTransactionOnRecompute();
static void removeTransactionOnRecompute();
static void setTransactionOnRecompute(const bool &v);
static const char *docTransactionOnRecompute();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter RelativeStringID
static const bool & getRelativeStringID();
static const bool & defaultRelativeStringID();
static void removeRelativeStringID();
static void setRelativeStringID(const bool &v);
static const char *docRelativeStringID();
//@}
// Auto generated code (Tools/params_utils.py:118)
//@{
/// Accessor for parameter EnableMaterialEdit
static const bool & getEnableMaterialEdit();
static const bool & defaultEnableMaterialEdit();
static void removeEnableMaterialEdit();
static void setEnableMaterialEdit(const bool &v);
static const char *docEnableMaterialEdit();
//@}
// Auto generated code (Tools/params_utils.py:146)
}; // class DocumentParams
} // namespace App
//[[[end]]]
#endif // DOCUMENT_PARAMS_H
src/App/StringHasher.cpp
+646
View File
@@ -0,0 +1,646 @@
/****************************************************************************
* Copyright (c) 2022 Zheng Lei (realthunder) <[email protected]> *
* *
* This file is part of the FreeCAD CAx development system. *
* *
* This library is free software; you can redistribute it and/or *
* modify it under the terms of the GNU Library General Public *
* License as published by the Free Software Foundation; either *
* version 2 of the License, or (at your option) any later version. *
* *
* This library is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Library General Public License for more details. *
* *
* You should have received a copy of the GNU Library General Public *
* License along with this library; see the file COPYING.LIB. If not, *
* write to the Free Software Foundation, Inc., 59 Temple Place, *
* Suite 330, Boston, MA 02111-1307, USA *
* *
****************************************************************************/
#include "PreCompiled.h"
#ifndef _PreComp_
#endif
#include <deque>
#include <boost/io/ios_state.hpp>
#include <boost/iostreams/device/array.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/classification.hpp>
#include <boost/bimap.hpp>
#include <boost/bimap/unordered_set_of.hpp>
#include <boost/bimap/set_of.hpp>
#include <QHash>
#include <QCryptographicHash>
#include <Base/Console.h>
#include <Base/Exception.h>
#include <Base/Writer.h>
#include <Base/Reader.h>
#include <App/StringHasher.h>
#include <App/StringHasherPy.h>
#include <App/StringIDPy.h>
#include <App/DocumentParams.h>
FC_LOG_LEVEL_INIT("App",true,true)
namespace bio = boost::iostreams;
using namespace App;
///////////////////////////////////////////////////////////
struct StringIDHasher {
std::size_t operator()(const StringID *sid) const {
if (!sid)
return 0;
return qHash(sid->data(), qHash(sid->postfix()));
}
bool operator()(const StringID *a, const StringID *b) const {
if (a == b)
return true;
if (!a || !b)
return false;
return a->data() == b->data() && a->postfix() == b->postfix();
}
};
typedef boost::bimap<
boost::bimaps::unordered_set_of<StringID*,
StringIDHasher,
StringIDHasher>,
boost::bimaps::set_of<long> >
HashMapBase;
class StringHasher::HashMap: public HashMapBase
{
public:
bool SaveAll = false;
int Threshold = 0;
};
///////////////////////////////////////////////////////////
TYPESYSTEM_SOURCE_ABSTRACT(App::StringID, Base::BaseClass)
StringID::~StringID()
{
if (_hasher)
_hasher->_hashes->right.erase(_id);
}
PyObject *StringID::getPyObject() {
return new StringIDPy(this);
}
PyObject *StringID::getPyObjectWithIndex(int index) {
auto res = new StringIDPy(this);
res->_index = index;
return res;
}
std::string StringID::toString(int index) const {
std::ostringstream ss;
ss << '#' << std::hex << value();
if (index)
ss << ':' << index;
return ss.str();
}
StringID::IndexID StringID::fromString(const char *name, bool eof, int size) {
IndexID res;
res.id = 0;
res.index = 0;
if (!name) {
res.id = -1;
return res;
}
if (size < 0)
size = std::strlen(name);
bio::stream<bio::array_source> iss(name, size);
char sep = 0;
char sep2 = 0;
iss >> sep >> std::hex >> res.id >> sep2 >> res.index;
if((eof && !iss.eof()) || sep!='#' || (sep2!=0 && sep2!=':')) {
res.id = -1;
return res;
}
return res;
}
std::string StringID::dataToText(int index) const {
if(isHashed() || isBinary())
return _data.toBase64().constData();
std::string res(_data.constData());
if (index)
res += std::to_string(index);
if (_postfix.size())
res += _postfix.constData();
return res;
}
void StringID::mark() const
{
if (isMarked())
return;
_flags.set(Marked);
for (auto & sid : _sids)
sid.deref().mark();
}
///////////////////////////////////////////////////////////
TYPESYSTEM_SOURCE(App::StringHasher, Base::Persistence)
StringHasher::StringHasher()
:_hashes(new HashMap)
{}
StringHasher::~StringHasher() {
clear();
}
void StringHasher::setSaveAll(bool enable) {
if (_hashes->SaveAll == enable)
return;
_hashes->SaveAll = enable;
compact();
}
void StringHasher::compact()
{
if (_hashes->SaveAll)
return;
std::deque<StringIDRef> pendings;
for (auto & v : _hashes->right) {
if (!v.second->isPersistent() && v.second->getRefCount() == 1)
pendings.emplace_back(v.second);
}
while(pendings.size()) {
StringIDRef sid = pendings.front();
pendings.pop_front();
if (!_hashes->right.erase(sid.value()))
continue;
sid._sid->_hasher = nullptr;
sid._sid->unref();
for (auto & s : sid._sid->_sids) {
if (s._sid->_hasher == this
&& !s._sid->isPersistent()
&& s._sid->getRefCount() == 2)
pendings.push_back(s);
}
}
}
bool StringHasher::getSaveAll() const {
return _hashes->SaveAll;
}
void StringHasher::setThreshold(int threshold) {
_hashes->Threshold = threshold;
}
int StringHasher::getThreshold() const {
return _hashes->Threshold;
}
long StringHasher::lastID() const {
if(_hashes->right.empty())
return 0;
auto it = _hashes->right.end();
--it;
return it->first;
}
StringIDRef StringHasher::getID(const char *text, int len, bool hashable) {
if(len<0) len = strlen(text);
return getID(QByteArray::fromRawData(text,len),false,hashable);
}
StringIDRef StringHasher::getID(const QByteArray &data, bool binary, bool hashable, bool nocopy)
{
bool hashed = hashable && _hashes->Threshold>0
&& (int)data.size()>_hashes->Threshold;
StringID d;
if(hashed) {
QCryptographicHash hasher(QCryptographicHash::Sha1);
hasher.addData(data);
d._data = hasher.result();
} else
d._data = data;
auto it = _hashes->left.find(&d);
if(it!=_hashes->left.end())
return StringIDRef(it->first);
if(!hashed && !nocopy) {
// if not hashed, make a deep copy of the data
d._data = QByteArray(data.constData(), data.size());
}
StringIDRef sid(new StringID(lastID()+1,d._data,binary,hashed));
return StringIDRef(insert(sid));
}
StringIDRef StringHasher::getID(long id, int index) const {
if(id<=0)
return StringIDRef();
auto it = _hashes->right.find(id);
if(it == _hashes->right.end())
return StringIDRef();
StringIDRef res(it->second);
res._index = index;
return res;
}
void StringHasher::setPersistenceFileName(const char *filename) const {
if(!filename)
filename = "";
_filename = filename;
}
const std::string &StringHasher::getPersistenceFileName() const {
return _filename;
}
void StringHasher::Save(Base::Writer &writer) const {
size_t count;
if (_hashes->SaveAll)
count = _hashes->size();
else {
count = 0;
for (auto & v : _hashes->right) {
if (v.second->isMarked() || v.second->isPersistent())
++count;
}
}
writer.Stream() << writer.ind()
<< "<StringHasher saveall=\"" << _hashes->SaveAll
<< "\" threshold=\"" << _hashes->Threshold << "\"";
if(!count) {
writer.Stream() << " count=\"0\"></StringHasher>\n";
return;
}
writer.Stream() << " count=\"0\" new=\"1\"/>\n";
writer.Stream() << writer.ind() << "<StringHasher2 ";
if(_filename.size()) {
writer.Stream() << " file=\""
<< writer.addFile(_filename+".txt",this)
<< "\"/>\n";
return;
}
writer.Stream() << " count=\"" << count << "\">\n";
saveStream(writer.beginCharStream(false) << '\n');
writer.endCharStream() << '\n';
writer.Stream() << writer.ind() << "</StringHasher2>\n";
}
void StringHasher::SaveDocFile (Base::Writer &writer) const {
std::size_t count = _hashes->SaveAll?this->size():this->count();
writer.Stream() << count << '\n';
saveStream(writer.Stream());
}
void StringHasher::saveStream(std::ostream &s) const {
Base::OutputStream str(s,false);
boost::io::ios_flags_saver ifs(s);
s << std::hex;
bool allowRealtive = DocumentParams::getRelativeStringID();
long anchor = 0;
const StringID *last = nullptr;
long lastid = 0;
bool relative = false;
for(auto &v : _hashes->right) {
auto & d = *v.second;
long id = d._id;
if (!_hashes->SaveAll && !d.isMarked() && !d.isPersistent())
continue;
if (!allowRealtive)
s << id;
else {
// We use relative coding to save space. But in order to have some
// minimum protection against corruption, write an absolute value every
// once a while.
relative = (id - anchor) < 1000;
if (relative)
s << '-' << id - lastid;
else {
anchor = id;
s << id;
}
lastid = id;
}
int offset = d.isPostfixEncoded() ? 1 : 0;
StringID::IndexID prefixid;
prefixid.id = 0;
prefixid.index = 0;
if (d.isPrefixID()) {
assert(d._sids.size() > offset);
prefixid.id = d._sids[offset].value();
}
else if (d.isPrefixIDIndex()) {
prefixid = StringID::fromString(d._data);
assert(d._sids.size() > offset && d._sids[offset].value() == prefixid.id);
}
auto flags = d._flags;
flags.reset(StringID::Marked);
s << '.' << flags.to_ulong();
int i = 0;
if (!relative) {
for (; i<d._sids.size(); ++i)
s << '.' << d._sids[i].value();
} else {
if (last) {
for (; i<d._sids.size() && i<last->_sids.size(); ++i) {
long m = last->_sids[i].value();
long n = d._sids[i].value();
if (n < m)
s << ".-" << m-n;
else
s << '.' << n-m;
}
}
for (; i<d._sids.size(); ++i)
s << '.' << id - d._sids[i].value();
}
last = & d;
// Having postfix means it is a geometry element name, which
// guarantees to be a single line without space. So it is safe to
// store in raw stream.
if (d.isPostfixed()) {
if (d.isPrefixIDIndex())
s << ' ' << prefixid.index;
else if (!d.isIndexed() && !d.isPrefixID())
s << ' ' << d._data.constData();
if (!d.isPostfixEncoded())
s << ' ' << d._postfix.constData();
s << '\n';
}
else {
// Reaching here means the string may contain space and newlines
// We rely on OutputStream (i.e. str) to save the string.
s << ' ';
str << d._data.constData();
}
}
}
void StringHasher::RestoreDocFile (Base::Reader &reader) {
std::string marker, ver;
reader >> marker;
std::size_t count;
_hashes->clear();
if (marker == "StringTableStart") {
reader >> ver >> count;
if (ver != "v1")
FC_WARN("Unknown string table format");
restoreStreamNew(reader, count);
return;
}
reader >> count;
restoreStream(reader,count);
}
void StringHasher::restoreStreamNew(std::istream &s, std::size_t count) {
Base::InputStream str(s,false);
_hashes->clear();
std::string content;
boost::io::ios_flags_saver ifs(s);
s >> std::hex;
std::vector<std::string> tokens;
long lastid = 0;
const StringID * last = nullptr;
std::string tmp;
for(uint32_t i=0;i<count;++i) {
if (!(s >> tmp))
FC_THROWM(Base::RuntimeError, "Invalid string table");
tokens.clear();
boost::split(tokens, tmp, boost::is_any_of("."));
if (tokens.size() < 2)
FC_THROWM(Base::RuntimeError, "Invalid string table");
long id;
bool relative = false;
if (tokens[0][0] == '-') {
relative = true;
id = lastid + strtol(tokens[0].c_str()+1, nullptr, 16);
} else
id = strtol(tokens[0].c_str(), nullptr, 16);
lastid = id;
unsigned long flag = strtol(tokens[1].c_str(), nullptr, 16);
StringIDRef sid(new StringID(id,QByteArray(),flag));
StringID & d = *sid._sid;
d._sids.reserve(tokens.size()-2);
int j = 2;
if (relative && last) {
for (;j<(int)tokens.size() && j-2<last->_sids.size(); ++j) {
long m = last->_sids[j-2].value();
long n;
if (tokens[j][0] == '-')
n = -strtol(&tokens[j][1], nullptr, 16);
else
n = strtol(&tokens[j][0], nullptr, 16);
StringIDRef sid = getID(m + n);
if (!sid)
FC_THROWM(Base::RuntimeError, "Invalid string id reference");
d._sids.push_back(sid);
}
}
for (;j<(int)tokens.size(); ++j) {
long n = strtol(&tokens[j][0], nullptr, 16) ;
StringIDRef sid = getID(relative ? id - n : n);
if (!sid)
FC_THROWM(Base::RuntimeError, "Invalid string id reference");
d._sids.push_back(sid);
}
if (!d.isPostfixed()) {
str >> content;
if(d.isHashed() || d.isBinary())
d._data = QByteArray::fromBase64(content.c_str());
else
d._data = content.c_str();
} else {
int offset = 0;
if (d.isPostfixEncoded()) {
offset = 1;
if (d._sids.empty())
FC_THROWM(Base::RuntimeError, "Missing string postfix");
d._postfix = d._sids[0]._sid->_data;
}
if (d.isIndexed()) {
if (d._sids.size() <= offset)
FC_THROWM(Base::RuntimeError, "Missing string prefix");
d._data = d._sids[offset]._sid->_data;
}
else if (d.isPrefixID() || d.isPrefixIDIndex()) {
if (d._sids.size() <= offset)
FC_THROWM(Base::RuntimeError, "Missing string prefix id");
int index = 0;
if (d.isPrefixIDIndex()) {
if (!(s >> index))
FC_THROWM(Base::RuntimeError, "Missing string prefix index");
}
d._data = d._sids[offset]._sid->toString(index).c_str();
} else {
s >> content;
d._data = content.c_str();
}
if (!d.isPostfixEncoded()) {
s >> content;
d._postfix = content.c_str();
}
}
last = insert(sid);
}
}
StringID * StringHasher::insert(const StringIDRef & sid)
{
assert(sid && sid._sid->_hasher == nullptr);
auto & d = *sid._sid;
d._hasher = this;
d.ref();
auto res = _hashes->right.insert(_hashes->right.end(),
HashMap::right_map::value_type(sid.value(),&d));
if (res->second != &d) {
d._hasher = nullptr;
d.unref();
}
return res->second;
}
void StringHasher::restoreStream(std::istream &s, std::size_t count) {
Base::InputStream str(s,false);
_hashes->clear();
std::string content;
for(uint32_t i=0;i<count;++i) {
int32_t id;
uint8_t type;
str >> id >> type >> content;
StringIDRef sid = new StringID(id,QByteArray(),type);
if(sid.isHashed() || sid.isBinary()) {
sid._sid->_data = QByteArray::fromBase64(content.c_str());
} else
sid._sid->_data = QByteArray(content.c_str());
insert(sid);
}
}
void StringHasher::clear() {
for (auto & v : _hashes->right) {
v.second->_hasher = nullptr;
v.second->unref();
}
_hashes->clear();
}
size_t StringHasher::size() const {
return _hashes->size();
}
size_t StringHasher::count() const {
size_t count = 0;
for(auto &v : _hashes->right)
if(v.second->getRefCount()>1)
++count;
return count;
}
void StringHasher::Restore(Base::XMLReader &reader) {
clear();
reader.readElement("StringHasher");
_hashes->SaveAll = reader.getAttributeAsInteger("saveall")?true:false;
_hashes->Threshold = reader.getAttributeAsInteger("threshold");
bool newtag = false;
if (reader.getAttributeAsInteger("new","0") > 0) {
reader.readElement("StringHasher2");
newtag = true;
}
if(reader.hasAttribute("file")) {
const char *file = reader.getAttribute("file");
if(*file)
reader.addFile(file,this);
return;
}
std::size_t count = reader.getAttributeAsUnsigned("count");
if (newtag) {
restoreStreamNew(reader.beginCharStream(false),count);
reader.readEndElement("StringHasher2");
return;
}
else if(count && reader.FileVersion > 1)
restoreStream(reader.beginCharStream(false),count);
else {
for(std::size_t i=0;i<count;++i) {
reader.readElement("Item");
StringIDRef sid;
long id = reader.getAttributeAsInteger("id");
bool hashed = reader.hasAttribute("hash");
if(hashed || reader.hasAttribute("data")) {
const char* value = hashed?reader.getAttribute("hash"):reader.getAttribute("data");
sid = new StringID(id,QByteArray::fromBase64(value),true,hashed);
}else {
sid = new StringID(id,QByteArray(reader.getAttribute("text")),false,false);
}
insert(sid);
}
}
reader.readEndElement("StringHasher");
}
unsigned int StringHasher::getMemSize (void) const {
return (_hashes->SaveAll?size():count()) * 10;
}
PyObject *StringHasher::getPyObject() {
return new StringHasherPy(this);
}
std::map<long,StringIDRef> StringHasher::getIDMap() const {
std::map<long,StringIDRef> ret;
for(auto &v : _hashes->right)
ret.emplace_hint(ret.end(), v.first, StringIDRef(v.second));
return ret;
}
void StringHasher::clearMarks() const
{
for (auto & v : _hashes->right)
v.second->_flags.reset(StringID::Marked);
}
src/App/StringHasher.h
+481
View File
@@ -0,0 +1,481 @@
/****************************************************************************
* Copyright (c) 2022 Zheng Lei (realthunder) <[email protected]> *
* *
* This file is part of the FreeCAD CAx development system. *
* *
* This library is free software; you can redistribute it and/or *
* modify it under the terms of the GNU Library General Public *
* License as published by the Free Software Foundation; either *
* version 2 of the License, or (at your option) any later version. *
* *
* This library is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Library General Public License for more details. *
* *
* You should have received a copy of the GNU Library General Public *
* License along with this library; see the file COPYING.LIB. If not, *
* write to the Free Software Foundation, Inc., 59 Temple Place, *
* Suite 330, Boston, MA 02111-1307, USA *
* *
****************************************************************************/
#ifndef APP_STRINGID_H
#define APP_STRINGID_H
#include <memory>
#include <bitset>
#include <QByteArray>
#include <QVector>
#include <CXX/Objects.hxx>
#include <Base/Handle.h>
#include <Base/Persistence.h>
namespace App {
class StringHasher;
class StringID;
class StringIDRef;
typedef Base::Reference<StringHasher> StringHasherRef;
class AppExport StringID: public Base::BaseClass, public Base::Handled {
TYPESYSTEM_HEADER_WITH_OVERRIDE();
public:
enum Flag {
Binary,
Hashed,
PostfixEncoded,
Postfixed,
Indexed,
PrefixID,
PrefixIDIndex,
Persistent,
Marked,
};
StringID(long id, const QByteArray &data, bool binary, bool hashed)
:_id(id),_data(data)
{
if(binary) _flags.set(Binary);
if(hashed) _flags.set(Hashed);
}
StringID(long id, const QByteArray &data, uint8_t flags)
:_id(id),_data(data),_flags(flags)
{}
StringID()
:_id(0),_flags(0)
{}
virtual ~StringID();
long value() const {return _id;}
const QVector<StringIDRef> &relatedIDs() const {return _sids;}
bool isBinary() const {return _flags.test(Binary);}
bool isHashed() const {return _flags.test(Hashed);}
bool isPostfixed() const {return _flags.test(Postfixed);}
bool isPostfixEncoded() const {return _flags.test(PostfixEncoded);}
bool isIndexed() const {return _flags.test(Indexed);}
bool isPrefixID() const {return _flags.test(PrefixID);}
bool isPrefixIDIndex() const {return _flags.test(PrefixIDIndex);}
bool isMarked() const {return _flags.test(Marked);}
bool isPersistent() const {return _flags.test(Persistent);}
bool isFromSameHasher(const StringHasherRef & hasher) const
{
return this->_hasher == hasher;
}
StringHasherRef getHasher() const
{
return StringHasherRef(_hasher);
}
const QByteArray data() const {return _data;}
const QByteArray postfix() const {return _postfix;}
virtual PyObject *getPyObject() override;
PyObject *getPyObjectWithIndex(int index);
std::string toString(int index) const;
struct IndexID {
long id;
int index;
explicit operator bool() const {
return id > 0;
}
friend std::ostream & operator << (std::ostream &s, const IndexID & id) {
s << id.id;
if (id.index)
s << ':' << id.index;
return s;
}
};
static IndexID fromString(const char *name, bool eof=true, int size = -1);
static IndexID fromString(const QByteArray &bytes, bool eof=true) {
return fromString(bytes.constData(), eof, bytes.size());
}
std::string dataToText(int index) const;
void toBytes(QByteArray &bytes, int index) const {
if (_postfix.size())
bytes = _data + _postfix;
else if (index)
bytes = _data + QByteArray::number(index);
else
bytes = _data;
}
void mark() const;
void setPersistent(bool enable)
{
_flags.set(Persistent, enable);
}
bool operator<(const StringID &other) const {
return compare(other) < 0;
}
int compare(const StringID &other) const {
if (_hasher < other._hasher)
return -1;
if (_hasher > other._hasher)
return 1;
if (_id < other._id)
return -1;
if (_id > other._id)
return 1;
return 0;
}
friend class StringHasher;
private:
long _id;
QByteArray _data;
QByteArray _postfix;
StringHasher *_hasher = nullptr;
mutable std::bitset<32> _flags;
mutable QVector<StringIDRef> _sids;
};
//////////////////////////////////////////////////////////////////////////
class StringIDRef
{
public:
StringIDRef()
:_sid(0), _index(0)
{}
StringIDRef(StringID* p, int index=0)
: _sid(p), _index(index)
{
if (_sid)
_sid->ref();
}
StringIDRef(const StringIDRef & other)
: _sid(other._sid)
, _index(other._index)
{
if (_sid)
_sid->ref();
}
StringIDRef(StringIDRef && other)
: _sid(other._sid)
, _index(other._index)
{
other._sid = nullptr;
}
StringIDRef(const StringIDRef & other, int index)
: _sid(other._sid)
, _index(index)
{
if (_sid)
_sid->ref();
}
~StringIDRef()
{
if (_sid)
_sid->unref();
}
void reset(const StringIDRef & p = StringIDRef()) {
*this = p;
}
void reset(const StringIDRef &p, int index) {
*this = p;
this->_index = index;
}
void swap(StringIDRef &p) {
if(*this != p) {
auto tmp = p;
p = *this;
*this = tmp;
}
}
StringIDRef & operator=(StringID* p) {
if (_sid == p)
return *this;
if (_sid)
_sid->unref();
_sid = p;
if (_sid)
_sid->ref();
this->_index = 0;
return *this;
}
StringIDRef & operator=(const StringIDRef & p) {
if (_sid != p._sid) {
if (_sid)
_sid->unref();
_sid = p._sid;
if (_sid)
_sid->ref();
}
this->_index = p._index;
return *this;
}
StringIDRef & operator=(StringIDRef && p) {
if (_sid != p._sid) {
if (_sid)
_sid->unref();
_sid = p._sid;
p._sid = nullptr;
}
this->_index = p._index;
return *this;
}
bool operator<(const StringIDRef & p) const {
if (!_sid)
return true;
if (!p._sid)
return false;
int res = _sid->compare(*p._sid);
if (res < 0)
return true;
if (res > 0)
return false;
return _index < p._index;
}
bool operator==(const StringIDRef & p) const {
return _sid == p._sid && _index == p._index;
}
bool operator!=(const StringIDRef & p) const {
return _sid != p._sid || _index != p._index;
}
explicit operator bool() const {
return _sid != nullptr;
}
int getRefCount(void) const {
if (_sid)
return _sid->getRefCount();
return 0;
}
std::string toString() const {
if (_sid)
return _sid->toString(_index);
return std::string();
}
std::string dataToText() const {
if (_sid)
return _sid->dataToText(_index);
return std::string();
}
const char * constData() const {
if (_sid) {
assert(_index == 0);
assert(_sid->postfix().isEmpty());
return _sid->data().constData();
}
return "";
}
const StringID & deref() const {
return *_sid;
}
long value() const {
if (_sid)
return _sid->value();
return 0;
}
QVector<StringIDRef> relatedIDs() const {
if (_sid)
return _sid->relatedIDs();
return QVector<StringIDRef>();
}
bool isBinary() const {
if (_sid)
return _sid->isBinary();
return false;
}
bool isHashed() const {
if (_sid)
return _sid->isHashed();
return false;
}
void toBytes(QByteArray &bytes) const {
if (_sid)
_sid->toBytes(bytes, _index);
}
PyObject *getPyObject(void) {
if (_sid)
return _sid->getPyObjectWithIndex(_index);
Py_INCREF(Py_None);
return Py_None;
}
void mark() const {
if (_sid)
_sid->mark();
}
bool isMarked() const {
return _sid && _sid->isMarked();
}
bool isFromSameHasher(const StringHasherRef & hasher) const
{
return _sid && _sid->isFromSameHasher(hasher);
}
StringHasherRef getHasher() const
{
if (_sid)
return _sid->getHasher();
return StringHasherRef();
}
void setPersistent(bool enable)
{
if (_sid)
_sid->setPersistent(enable);
}
friend class StringHasher;
private:
StringID *_sid;
int _index;
};
/// A String table to map string from/to a unique integer
class AppExport StringHasher: public Base::Persistence, public Base::Handled {
TYPESYSTEM_HEADER_WITH_OVERRIDE();
public:
StringHasher();
virtual ~StringHasher();
virtual unsigned int getMemSize (void) const override;
virtual void Save (Base::Writer &/*writer*/) const override;
virtual void Restore(Base::XMLReader &/*reader*/) override;
virtual void SaveDocFile (Base::Writer &/*writer*/) const override;
virtual void RestoreDocFile (Base::Reader &/*reader*/) override;
void setPersistenceFileName(const char *name) const;
const std::string &getPersistenceFileName() const;
/** Maps an arbitrary string to an integer
*
* The function maps an arbitrary text string to a unique integer ID, which
* is returned as a shared pointer to reference count the ID so that it is
* possible to prune any unused strings.
*
* If the string is longer than the threshold setting of this StringHasher,
* it will be sha1 hashed before storing, and the original content of the
* string is discarded.
*
* The purpose of function is to provide a short form of a stable string
* identification.
*/
StringIDRef getID(const char *text, int len=-1, bool hashable=false);
/** Map text or binary data to an integer */
StringIDRef getID(const QByteArray & data, bool binary, bool hashable=true, bool nocopy=false);
/** Obtain the reference counted StringID object from numerical id
*
* This function exists because the stored string may be one way hashed,
* and the original text is not persistent. The caller use this function to
* retrieve the reference count ID object after restore
*/
StringIDRef getID(long id, int index = 0) const;
StringIDRef getID(const StringID::IndexID &id) const {
return getID(id.id, id.index);
}
std::map<long,StringIDRef> getIDMap() const;
/// Clear all string hashes
void clear();
/// Size of the hash table
size_t size() const;
/// Return the number of hashes that are used by others
size_t count() const;
virtual PyObject *getPyObject(void) override;
void setSaveAll(bool enable);
bool getSaveAll() const;
void setThreshold(int threshold);
int getThreshold() const;
void clearMarks() const;
void compact();
class HashMap;
friend class StringID;
protected:
StringID * insert(const StringIDRef & sid);
long lastID() const;
void saveStream(std::ostream &s) const;
void restoreStream(std::istream &s, std::size_t count);
void restoreStreamNew(std::istream &s, std::size_t count);
private:
std::unique_ptr<HashMap> _hashes;
mutable std::string _filename;
};
}
#endif
src/App/StringHasherPy.xml
+70
View File
@@ -0,0 +1,70 @@
<?xml version="1.0" encoding="UTF-8"?>
<GenerateModel xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="generateMetaModel_Module.xsd">
<PythonExport
Father="BaseClassPy"
Name="StringHasherPy"
Twin="StringHasher"
TwinPointer="StringHasher"
Include="App/StringHasher.h"
FatherInclude="Base/BaseClassPy.h"
Namespace="App"
FatherNamespace="Base"
Constructor="true"
Reference="true">
<Documentation>
<Author Licence="LGPL" Name="Zheng, Lei" EMail="[email protected]" />
<DeveloperDocu>This is the StringHasher class</DeveloperDocu>
<UserDocu>This is the StringHasher class</UserDocu>
</Documentation>
<Methode Name="getID">
<Documentation>
<UserDocu>
getID(txt|id, base64=False) -> StringID
If the input is text, return a StringID object that is unique within this hasher. This
StringID object is reference counted. The hasher may only save hash ID's that are used.
If the input is an integer, then the hasher will try to find the StringID object stored
with the same integer value.
base64: indicate if the input 'txt' is base64 encoded binary data
</UserDocu>
</Documentation>
</Methode>
<Methode Name="isSame" Const="true">
<Documentation>
<UserDocu>Check if two hasher are the same</UserDocu>
</Documentation>
</Methode>
<Attribute Name="Count" ReadOnly="true">
<Documentation>
<UserDocu>Return count of used hashes</UserDocu>
</Documentation>
<Parameter Name="Count" Type="Int" />
</Attribute>
<Attribute Name="Size" ReadOnly="true">
<Documentation>
<UserDocu>Return the size of the hashes</UserDocu>
</Documentation>
<Parameter Name="Size" Type="Int"/>
</Attribute>
<Attribute Name="SaveAll">
<Documentation>
<UserDocu>Whether to save all string hashes regardless of its use count</UserDocu>
</Documentation>
<Parameter Name="SaveAll" Type="Boolean"/>
</Attribute>
<Attribute Name="Threshold">
<Documentation>
<UserDocu>Data length exceed this threshold will be hashed before storing</UserDocu>
</Documentation>
<Parameter Name="Threshold" Type="Int"/>
</Attribute>
<Attribute Name="Table" ReadOnly="true">
<Documentation>
<UserDocu>Return the entire string table as Int->String dictionary</UserDocu>
</Documentation>
<Parameter Name="Table" Type="Dict"/>
</Attribute>
</PythonExport>
</GenerateModel>
src/App/StringHasherPyImp.cpp
+150
View File
@@ -0,0 +1,150 @@
/****************************************************************************
* Copyright (c) 2018 Zheng Lei (realthunder) <[email protected]> *
* *
* This file is part of the FreeCAD CAx development system. *
* *
* This library is free software; you can redistribute it and/or *
* modify it under the terms of the GNU Library General Public *
* License as published by the Free Software Foundation; either *
* version 2 of the License, or (at your option) any later version. *
* *
* This library is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Library General Public License for more details. *
* *
* You should have received a copy of the GNU Library General Public *
* License along with this library; see the file COPYING.LIB. If not, *
* write to the Free Software Foundation, Inc., 59 Temple Place, *
* Suite 330, Boston, MA 02111-1307, USA *
* *
****************************************************************************/
#include "PreCompiled.h"
#include "StringHasher.h"
#include "StringHasherPy.h"
#include "StringHasherPy.cpp"
using namespace App;
// returns a string which represent the object e.g. when printed in python
std::string StringHasherPy::representation(void) const
{
std::ostringstream str;
str << "<StringHasher at " << getStringHasherPtr() << ">";
return str.str();
}
PyObject *StringHasherPy::PyMake(struct _typeobject *, PyObject *, PyObject *) // Python wrapper
{
return new StringHasherPy(new StringHasher);
}
// constructor method
int StringHasherPy::PyInit(PyObject* , PyObject* )
{
return 0;
}
PyObject* StringHasherPy::isSame(PyObject *args)
{
PyObject *other;
if (!PyArg_ParseTuple(args, "O!", &StringHasherPy::Type, &other)){ // convert args: Python->C
return Py::new_reference_to(Py::False());
}
auto otherHasher = static_cast<StringHasherPy*>(other)->getStringHasherPtr();
return Py::new_reference_to(Py::Boolean(getStringHasherPtr() == otherHasher));
}
PyObject* StringHasherPy::getID(PyObject *args)
{
long id = -1;
int index = 0;
PyObject *value = 0;
PyObject *base64 = Py_False;
if (!PyArg_ParseTuple(args, "l|i",&id,&index)) {
PyErr_Clear();
if (!PyArg_ParseTuple(args, "O|O",&value,&base64))
return NULL; // NULL triggers exception
}
if(id>0) {
PY_TRY {
auto sid = getStringHasherPtr()->getID(id, index);
if(!sid) Py_Return;
return sid.getPyObject();
}PY_CATCH;
}
std::string txt;
#if PY_MAJOR_VERSION >= 3
if (PyUnicode_Check(value)) {
txt = PyUnicode_AsUTF8(value);
}
#else
if (PyUnicode_Check(value)) {
PyObject* unicode = PyUnicode_AsLatin1String(value);
txt = PyString_AsString(unicode);
Py_DECREF(unicode);
}
else if (PyString_Check(value)) {
txt = PyString_AsString(value);
}
#endif
else
throw Py::TypeError("expect argument of type string");
PY_TRY {
QByteArray data;
StringIDRef sid;
if(PyObject_IsTrue(base64)) {
data = QByteArray::fromBase64(QByteArray::fromRawData(txt.c_str(),txt.size()));
sid = getStringHasherPtr()->getID(data,true);
}else
sid = getStringHasherPtr()->getID(txt.c_str(),txt.size());
return sid.getPyObject();
}PY_CATCH;
}
Py::Int StringHasherPy::getCount(void) const {
return Py::Int((long)getStringHasherPtr()->count());
}
Py::Int StringHasherPy::getSize(void) const {
return Py::Int((long)getStringHasherPtr()->size());
}
Py::Boolean StringHasherPy::getSaveAll(void) const {
return Py::Boolean(getStringHasherPtr()->getSaveAll());
}
void StringHasherPy::setSaveAll(Py::Boolean value) {
getStringHasherPtr()->setSaveAll(value);
}
Py::Int StringHasherPy::getThreshold(void) const {
return Py::Int((long)getStringHasherPtr()->getThreshold());
}
void StringHasherPy::setThreshold(Py::Int value) {
getStringHasherPtr()->setThreshold(value);
}
Py::Dict StringHasherPy::getTable() const {
Py::Dict dict;
for(auto &v : getStringHasherPtr()->getIDMap())
dict.setItem(Py::Int(v.first),Py::String(v.second.dataToText()));
return dict;
}
PyObject *StringHasherPy::getCustomAttributes(const char* /*attr*/) const
{
return 0;
}
int StringHasherPy::setCustomAttributes(const char* /*attr*/, PyObject* /*obj*/)
{
return 0;
}
src/App/StringIDPy.xml
+58
View File
@@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<GenerateModel xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="generateMetaModel_Module.xsd">
<PythonExport
Father="BaseClassPy"
Name="StringIDPy"
Twin="StringID"
TwinPointer="StringID"
Include="App/StringHasher.h"
FatherInclude="Base/BaseClassPy.h"
Namespace="App"
FatherNamespace="Base"
Reference="true">
<Documentation>
<Author Licence="LGPL" Name="Zheng, Lei" EMail="[email protected]" />
<DeveloperDocu>This is the StringID class</DeveloperDocu>
<UserDocu>This is the StringID class</UserDocu>
</Documentation>
<Methode Name="isSame" Const="true">
<Documentation>
<UserDocu>Check if two StringIDs are the same</UserDocu>
</Documentation>
</Methode>
<Attribute Name="Value" ReadOnly="true">
<Documentation>
<UserDocu>Return the integer value of this ID</UserDocu>
</Documentation>
<Parameter Name="Value" Type="Int"/>
</Attribute>
<Attribute Name="Data" ReadOnly="true">
<Documentation>
<UserDocu>Return the data associated with this ID</UserDocu>
</Documentation>
<Parameter Name="Data" Type="String"/>
</Attribute>
<Attribute Name="IsBinary" ReadOnly="true">
<Documentation>
<UserDocu>Check if the data is binary, </UserDocu>
</Documentation>
<Parameter Name="IsBinary" Type="Boolean"/>
</Attribute>
<Attribute Name="IsHashed" ReadOnly="true">
<Documentation>
<UserDocu>Check if the data is hash, if so 'Data' returns a base64 encoded string of the raw hash</UserDocu>
</Documentation>
<Parameter Name="IsHashed" Type="Boolean"/>
</Attribute>
<Attribute Name="Index" ReadOnly="false">
<Documentation>
<UserDocu>Geometry index. Only meaningful for geometry element name</UserDocu>
</Documentation>
<Parameter Name="Index" Type="Int"/>
</Attribute>
<ClassDeclarations>private:
friend class StringID;
int _index = 0;
</ClassDeclarations>
</PythonExport>
</GenerateModel>
src/App/StringIDPyImp.cpp
+84
View File
@@ -0,0 +1,84 @@
/****************************************************************************
* Copyright (c) 2018 Zheng Lei (realthunder) <[email protected]> *
* *
* This file is part of the FreeCAD CAx development system. *
* *
* This library is free software; you can redistribute it and/or *
* modify it under the terms of the GNU Library General Public *
* License as published by the Free Software Foundation; either *
* version 2 of the License, or (at your option) any later version. *
* *
* This library is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Library General Public License for more details. *
* *
* You should have received a copy of the GNU Library General Public *
* License along with this library; see the file COPYING.LIB. If not, *
* write to the Free Software Foundation, Inc., 59 Temple Place, *
* Suite 330, Boston, MA 02111-1307, USA *
* *
****************************************************************************/
#include "PreCompiled.h"
#include "StringHasher.h"
#include "StringIDPy.h"
#include "StringIDPy.cpp"
using namespace App;
// returns a string which represent the object e.g. when printed in python
std::string StringIDPy::representation(void) const
{
return getStringIDPtr()->toString(_index);
}
PyObject* StringIDPy::isSame(PyObject *args)
{
PyObject *other;
if (!PyArg_ParseTuple(args, "O!", &StringIDPy::Type, &other)) { // convert args: Python->C
return Py::new_reference_to(Py::False());
}
auto otherPy = static_cast<StringIDPy*>(other);
return Py::new_reference_to(Py::Boolean(
otherPy->getStringIDPtr() == this->getStringIDPtr()
&& otherPy->_index == this->_index));
}
Py::Int StringIDPy::getValue(void) const {
return Py::Int(getStringIDPtr()->value());
}
Py::String StringIDPy::getData(void) const {
return Py::String(getStringIDPtr()->dataToText(this->_index));
}
Py::Boolean StringIDPy::getIsBinary(void) const {
return Py::Boolean(getStringIDPtr()->isBinary());
}
Py::Boolean StringIDPy::getIsHashed(void) const {
return Py::Boolean(getStringIDPtr()->isHashed());
}
Py::Int StringIDPy::getIndex(void) const {
return Py::Int(this->_index);
}
void StringIDPy::setIndex(Py::Int index) {
this->_index = index;
}
PyObject *StringIDPy::getCustomAttributes(const char* /*attr*/) const
{
return 0;
}
int StringIDPy::setCustomAttributes(const char* /*attr*/, PyObject* /*obj*/)
{
return 0;
}