62 #ifndef __ARC_DATAPOINT_H__ 63 #define __ARC_DATAPOINT_H__ 69 #include <arc/DateTime.h> 71 #include <arc/UserConfig.h> 72 #include <arc/data/DataStatus.h> 73 #include <arc/data/FileInfo.h> 74 #include <arc/data/URLMap.h> 75 #include <arc/loader/Loader.h> 76 #include <arc/loader/Plugin.h> 218 virtual std::string
str()
const;
221 virtual operator bool()
const;
250 unsigned int& wait_time);
276 unsigned int& wait_time);
404 const std::list<DataPoint*>& urls,
488 virtual void Passive(
bool v) = 0;
503 virtual void Range(
unsigned long long int start = 0,
504 unsigned long long int end = 0) = 0;
580 virtual void SetSize(
const unsigned long long int val);
583 virtual unsigned long long int GetSize()
const;
622 virtual long long int BufSize()
const = 0;
625 virtual int BufNum()
const = 0;
628 virtual bool Cache()
const;
631 virtual bool Local()
const = 0;
658 virtual bool IsIndex()
const = 0;
757 const std::string& meta) = 0;
788 const URLMap& url_map) = 0;
795 virtual void AddURLOptions(
const std::map<std::string, std::string>& options);
852 class DataPointLoader
863 class DataPointPluginArgument
864 :
public PluginArgument {
866 DataPointPluginArgument(
const URL& url,
const UserConfig& usercfg)
869 ~DataPointPluginArgument() {}
870 operator const URL&() {
873 operator const UserConfig&() {
878 const UserConfig& usercfg;
884 #endif // __ARC_DATAPOINT_H__ Arc namespace contains all core ARC classes.
Definition: ArcConfig.h:11
virtual bool Local() const =0
Returns true if file is local, e.g. file:// urls.
virtual void SetCheckSum(const std::string &val)
Set value of meta-information 'checksum'.
virtual bool SetURL(const URL &url)
Assigns new URL.
User configuration class
Definition: UserConfig.h:196
virtual DataStatus CompareLocationMetadata() const =0
Compare metadata of DataPoint and current location.
virtual bool LocationValid() const =0
Returns false no more locations are left and out of retries.
virtual const CheckSum * GetCheckSumObject(int index) const =0
Get CheckSum object at given position in list.
virtual const URL & CurrentLocation() const =0
Returns current (resolved) URL.
virtual DataPoint * CurrentLocationHandle() const =0
Returns a pointer to the DataPoint representing the current location.
virtual bool CheckSize() const
Check if meta-information 'size' is available.
virtual DataStatus AddLocation(const URL &url, const std::string &meta)=0
Add URL representing physical replica to list of locations.
DataPointAccessLatency
Describes the latency to access this URL.
Definition: DataPoint.h:140
This class is a wrapper around the DataPoint class.
Definition: DataHandle.h:33
virtual bool HaveLocations() const =0
Returns true if number of resolved URLs is not 0.
virtual bool SupportsTransfer() const
Returns true if DataPoint supports internal transfer.
virtual DataStatus GetFailureReason(void) const
Returns reason of transfer failure, as reported by callbacks.
A class for storing and manipulating times.
Definition: DateTime.h:125
virtual void Range(unsigned long long int start=0, unsigned long long int end=0)=0
Set range of bytes to retrieve.
virtual DataStatus Unregister(bool all)=0
Index service unregistration.
bool cache
Whether this DataPoint is cacheable.
Definition: DataPoint.h:819
Metadata describing content, like size, etc.
Definition: DataPoint.h:155
virtual void SetMeta(const DataPoint &p)
Copy meta information from another object.
URL url
URL supplied in constructor.
Definition: DataPoint.h:799
virtual bool RequiresCredentials() const
Returns true if some kind of credentials are needed to use this DataPoint.
virtual ~DataPoint()
Destructor.
Status code returned by many DataPoint methods.
Definition: DataStatus.h:54
static DataStatus Transfer3rdParty(const URL &source, const URL &destination, const UserConfig &usercfg, TransferCallback callback=NULL)
Perform third party transfer.
virtual bool Registered() const =0
Returns true if file is registered in indexing service (only known after Resolve()) ...
virtual const std::string & CurrentLocationMetadata() const =0
Returns meta information used to create current URL.
virtual bool Cache() const
Returns true if file is cacheable.
virtual const Time & GetModified() const
Get value of meta-information 'modification time'.
virtual void SetTries(const int n)
Set number of retries.
virtual bool IsIndex() const =0
Check if URL is an Indexing Service.
virtual DataStatus StartWriting(DataBuffer &buffer, DataCallback *space_cb=NULL)=0
Start writing data to URL.
A logger class.
Definition: Logger.h:493
virtual const std::string & GetCheckSum() const
Get value of meta-information 'checksum'.
virtual const URL & GetURL() const
Returns the URL that was passed to the constructor.
virtual DataStatus PreRegister(bool replication, bool force=false)=0
Index service pre-registration.
virtual bool NextLocation()=0
Switch to next location in list of URLs.
virtual const Time & GetValid() const
Get value of meta-information 'validity time'.
Access control - ownership, permission, etc.
Definition: DataPoint.h:157
virtual void SetSecure(bool v)=0
Allow/disallow heavy security (data encryption) during data transfer.
URL can be accessed instantly.
Definition: DataPoint.h:142
virtual DataStatus StartReading(DataBuffer &buffer)=0
Start reading data from URL.
virtual DataStatus Resolve(bool source)=0
Resolves index service URL into list of ordinary URLs.
virtual DataStatus RemoveLocations(const DataPoint &p)=0
Remove locations present in another DataPoint object.
virtual int AddCheckSumObject(CheckSum *cksum)=0
Add a checksum object which will compute checksum during data transfer.
virtual bool IsStageable() const
Check if URL should be staged or queried for Transport URL (TURL)
unsigned long long int size
Size of object represented by DataPoint.
Definition: DataPoint.h:805
virtual DataStatus CreateDirectory(bool with_parents=false)=0
Create a directory.
A DataPoint represents a data resource and is an abstraction of a URL.
Definition: DataPoint.h:121
virtual void ResetMeta()
Reset meta information to default (undefined) values.
std::set< std::string > valid_url_options
Valid URL options. Subclasses should add their own specific options to this list. ...
Definition: DataPoint.h:823
virtual std::string str() const
Returns a string representation of the DataPoint.
virtual bool CheckCheckSum() const
Check if meta-information 'checksum' is available.
Timestamps associated with object.
Definition: DataPoint.h:154
virtual DataStatus PrepareWriting(unsigned int timeout, unsigned int &wait_time)
Prepare DataPoint for writing.
const UserConfig usercfg
UserConfig supplied in constructor.
Definition: DataPoint.h:801
Time valid
Validity time of object represented by DataPoint.
Definition: DataPoint.h:811
Interface for checksum manipulations.
Definition: CheckSum.h:30
virtual DataStatus RemoveLocation()=0
Remove current URL from list.
virtual std::vector< URL > TransferLocations() const
Returns physical file(s) to read/write, if different from CurrentLocation()
virtual bool GetAdditionalChecks() const =0
Returns true unless SetAdditionalChecks() was set to false.
DataStatus failure_code
Result of data read/write carried out in separate thread.
Definition: DataPoint.h:817
virtual DataStatus Transfer(const URL &otherendpoint, bool source, TransferCallback callback=NULL)
Do a transfer in a single operation.
virtual void ReadOutOfOrder(bool v)=0
Allow/disallow DataPoint to read data out of order.
Time modified
Modification time of object represented by DataPoint.
Definition: DataPoint.h:809
Only name of object (relative).
Definition: DataPoint.h:152
virtual bool CheckModified() const
Check if meta-information 'modification time' is available.
virtual const UserConfig & GetUserConfig() const
Returns the UserConfig that was passed to the constructor.
virtual DataStatus Check(bool check_meta)=0
Query the DataPoint to check if object is accessible.
virtual DataStatus Remove()=0
Remove/delete object at URL.
virtual DataStatus Stat(FileInfo &file, DataPointInfoType verb=INFO_TYPE_ALL)=0
Retrieve information about this object.
virtual DataStatus PreUnregister(bool replication)=0
Index service pre-unregistration.
Fine structure - replicas, transfer locations, redirections.
Definition: DataPoint.h:158
virtual DataStatus StopWriting()=0
Stop writing.
URL has low (but non-zero) access latency, for example staged from disk.
Definition: DataPoint.h:144
Represents set of buffers.
Definition: DataBuffer.h:19
virtual DataStatus PostRegister(bool replication)=0
Index service post-registration.
virtual void SetAccessLatency(const DataPointAccessLatency &latency)
Set value of meta-information 'access latency'.
virtual void SetSize(const unsigned long long int val)
Set value of meta-information 'size'.
virtual bool AcceptsMeta() const =0
Check if endpoint can have any use from meta information.
virtual unsigned long long int GetSize() const
Get value of meta-information 'size'.
Metadata describing checksum.
Definition: DataPoint.h:156
Class to represent general URLs.
Definition: URL.h:88
virtual DataStatus PrepareReading(unsigned int timeout, unsigned int &wait_time)
Prepare DataPoint for reading.
virtual void AddURLOptions(const std::map< std::string, std::string > &options)
Add URL options to this DataPoint's URL object.
virtual int BufNum() const =0
Get suggested number of buffers for transfers.
virtual bool operator!() const
Is DataPoint valid?
virtual long long int BufSize() const =0
Get suggested buffer size for transfers.
int triesleft
Retries left for data transfer.
Definition: DataPoint.h:815
virtual void NextTry()
Decrease number of retries left.
virtual bool LastLocation()=0
Returns true if the current location is the last.
virtual bool CheckValid() const
Check if meta-information 'validity time' is available.
DataPointInfoType
Describes type of information about URL to request.
Definition: DataPoint.h:150
virtual int GetTries() const
Returns number of retries left.
All the other parameters.
Definition: DataPoint.h:159
virtual DataPointAccessLatency GetAccessLatency() const
Get value of meta-information 'access latency'.
virtual void SetValid(const Time &val)
Set value of meta-information 'validity time'.
URLMap allows mapping certain patterns of URLs to other URLs.
Definition: URLMap.h:22
static Logger logger
Logger object.
Definition: DataPoint.h:826
virtual bool ProvidesMeta() const =0
Check if endpoint can provide at least some meta information directly.
virtual bool CompareMeta(const DataPoint &p) const
Compare meta information from another object.
DataPointAccessLatency access_latency
Access latency of object represented by DataPoint.
Definition: DataPoint.h:813
Callbacks to be used when there is not enough space on the local filesystem.
Definition: DataCallback.h:20
virtual DataStatus FinishReading(bool error=false)
Finish reading from the URL.
virtual void SortLocations(const std::string &pattern, const URLMap &url_map)=0
Sort locations according to the specified pattern and URLMap.
virtual DataStatus ClearLocations()=0
Remove all locations.
void(* TransferCallback)(unsigned long long int bytes_transferred)
Callback for use in protocol-internal or 3rd party transfers.
Definition: DataPoint.h:131
virtual void Passive(bool v)=0
Set passive transfers for FTP-like protocols.
virtual DataStatus FinishWriting(bool error=false)
Finish writing to the URL.
All the parameters.
Definition: DataPoint.h:160
virtual bool GetSecure() const =0
Returns true if heavy security during data transfer is allowed.
virtual bool ReadOnly() const =0
Returns true if file is readonly.
DataPoint(const URL &url, const UserConfig &usercfg, PluginArgument *parg)
Constructor.
virtual void ClearTransferLocations()
Clear list of physical file(s) to read/write.
Definition: DataPoint.h:711
virtual DataStatus List(std::list< FileInfo > &files, DataPointInfoType verb=INFO_TYPE_ALL)=0
List hierarchical content of this object.
URL has a large access latency, for example staged from tape.
Definition: DataPoint.h:146
virtual DataStatus StopReading()=0
Stop reading.
Whatever protocol can get with no additional effort.
Definition: DataPoint.h:151
Type of object - currently file or dir.
Definition: DataPoint.h:153
bool stageable
Whether this DataPoint requires staging.
Definition: DataPoint.h:821
std::string checksum
Checksum of object represented by DataPoint.
Definition: DataPoint.h:807
FileInfo stores information about files (metadata).
Definition: FileInfo.h:25
virtual void SetAdditionalChecks(bool v)=0
Allow/disallow additional checks on a source DataPoint before transfer.
virtual void SetModified(const Time &val)
Set value of meta-information 'modification time'.
virtual bool RequiresCredentialsInFile() const
Returns true if credentials must be stored in files.
virtual const std::string DefaultCheckSum() const
Default checksum type (varies by protocol)
virtual DataStatus Rename(const URL &newurl)=0
Rename a URL.
virtual bool WriteOutOfOrder() const =0
Returns true if DataPoint supports receiving data out of order during writing.