/************************************************************************
*时间：2021/06/29
*类介绍：http 下载类
************************************************************************/
#ifndef DOWNLOADER_H
#define DOWNLOADER_H
#include <string>
#include <queue>
#include <QFile>
#include <QMutex>
#include <QNetworkReply>
#include <QThread>
/** \class DataReceiver
*
* 数据接收者抽象基类
*/
class DataReceiver {
public:
virtual ~DataReceiver();
enum DataMode {
DM_MEMORY = 0, ///< 下载到内存
DM_FILE, ///< 下载到文件
};
public:
virtual const std::string &getLocalDataPath(void) const = 0;
virtual void receiveData(QByteArray data) = 0;
void setMode(DataMode mode);
DataMode mode(void) const;
protected:
DataMode _dataMode = DM_FILE;
};
/** class NetFile
*
* 远程文件对象
*/
class NetFile : public DataReceiver {
public:
NetFile();
NetFile(const QString &name, const QString path="", DataMode mode = DM_FILE);
virtual ~NetFile();
const std::string &getLocalDataPath(void) const override;
void receiveData(QByteArray data);
const QByteArray getData(void);
QString getName(void);
QString getPath(void);;
void close(void);
protected:
QByteArray _data;
QFile _file;
QString _name;
QString _path;
};
/** class DataDownloadWorker
*
* 下载器工作线程
*/
class DataDownloadWorker : public QThread {
Q_OBJECT
public:
typedef struct {
std::string _url;
DataReceiver *_receiver;
bool _ignore;
} Task;
public:
DataDownloadWorker();
~DataDownloadWorker();
public:
/** fn Download
*
* 从指定URL下载文件，将数据存入DataReceiver对象
*
* param url 待下载文件的远程路径
* param receiver 一个用于接收数据的DataReceiver对象
*/
void DownLoadTo(const std::string &url, DataReceiver *receiver);
bool isIgnorable(const std::string &url);
/** fn removeTask
*
* 从下载队列删除任务
*
* param url 下载队列中的任务远程ＵＲＬ
*
* remark 如果数据已经全部下载成功，此函数不会删除已经下载的数据
*/
void removeTask(const std::string &url);
qint64 currentJobCount(void) const;
signals:
void downloadingProgress(int progress);
void NetFileDownLoaded(NetFile *);
void startJob(QString);
void networkError(QString msg, QString file);
public slots:
void replyFinished(QNetworkReply *reply);
void onTransferring(qint64, qint64);
void onStartJob(QString);
private:
void run() override;
private:
QNetworkAccessManager *_net = nullptr;
std::map<std::string, DataReceiver *> _task_paths;
std::queue<Task> _task_queue;
bool _running = false;
QMutex _stl_mutex, _downloadingMutex;
};
#endif // DOWNLOADER_H
#include <QDir>
#include <iostream>
#include <QTextCodec>
#include "DownLoader.h"
using namespace std;
DataReceiver::~DataReceiver() {
}
void DataReceiver::setMode(DataMode mode) {
this->_dataMode = mode;
}
DataReceiver::DataMode DataReceiver::mode() const {
return _dataMode;
}
NetFile::NetFile(const QString &name, const QString path/* ="" */, DataMode mode /* = DM_FILE */) :
_file(path +"/"+ name)
{
_name = name;
_dataMode = mode;
_path = path;
QDir outDir(path);
if ( _dataMode == DM_FILE && !outDir.exists() )
outDir.mkpath(path);
if ( _dataMode == DM_FILE )
_file.open(QIODevice::ReadWrite);
}
NetFile::NetFile() {
_dataMode = DM_MEMORY;
}
NetFile::~NetFile() {
if ( _file.isOpen() )
_file.close();
}
const QByteArray NetFile::getData() {
if ( _dataMode == DM_FILE && _data.isEmpty() )
_data = _file.readAll();
return _data;
}
QString NetFile::getName() {
return _file.fileName();
}
QString NetFile::getPath(void)
{
return _path;
}
void NetFile::close() {
if ( _dataMode == DM_FILE && _file.isOpen() )
_file.close();
}
const std::string &NetFile::getLocalDataPath(void) const {
static string result;
result.clear();
if ( _dataMode == DM_FILE )
result = _file.fileName().toStdString();
return result;
}
void NetFile::receiveData(QByteArray data)
{
if (_dataMode == DM_FILE)
{
_file.atEnd();
_file.write(data);
}
}
DataDownloadWorker::DataDownloadWorker() {
#ifdef _DEBUG
this->setObjectName("_file_downloader");
#endif
_net = new QNetworkAccessManager;
connect(_net, SIGNAL(finished(QNetworkReply*)),
this, SLOT(replyFinished(QNetworkReply*)));
connect(this, SIGNAL(startJob(QString)),
this, SLOT(onStartJob(QString)));
_running = true;
this->start();
}
DataDownloadWorker::~DataDownloadWorker() {
_running = false;
quit();
wait();
if ( _net )
delete _net, _net = nullptr;
}
qint64 DataDownloadWorker::currentJobCount() const {
return _task_paths.size();
}
void DataDownloadWorker::removeTask(const string &url) {
_stl_mutex.lock();
if ( _task_paths.find(url) != _task_paths.end() ) {
queue<Task> tmp;
while ( !_task_queue.empty() ) {
if ( _task_queue.front()._url != url )
tmp.push(_task_queue.front());
_task_queue.pop();
}
while ( !tmp.empty() ) {
_task_queue.push(tmp.front());
tmp.pop();
}
_task_paths.erase(url);
}
_stl_mutex.unlock();
}
bool DataDownloadWorker::isIgnorable(const string &url) {
bool result = false;
size_t size = url.size();
if ( size > 1 ) {
const char *str = url.c_str();
if (str[size-1] == 'a' && str[size-2] == '/' )
result = true;
}
return result;
}
void DataDownloadWorker::DownLoadTo(const string &url, DataReceiver *recv) {
_stl_mutex.lock();
if ( _task_paths.find(url) == _task_paths.end() ) {
Task task = { url, recv, false };
_task_paths.insert(std::pair<string, DataReceiver *>(url, recv));
_task_queue.push(task);
}
_stl_mutex.unlock();
}
void DataDownloadWorker::onStartJob(QString path) {
auto reply = _net->get( QNetworkRequest( QUrl(path) ) );
connect(reply, SIGNAL(downloadProgress(qint64,qint64)),
this, SLOT(onTransferring(qint64,qint64)));
}
void DataDownloadWorker::onTransferring(qint64 size_read, qint64 total_size) {
int percent = total_size ? size_read * 100 / total_size : 0;
emit downloadingProgress(percent);
}
void DataDownloadWorker::run() {
while ( _running ) {
if ( _stl_mutex.tryLock() ) {
if ( !_task_queue.empty() ) {
if ( _downloadingMutex.tryLock() ) {
if ( _task_queue.front()._ignore )
_task_queue.pop();
emit startJob(_task_queue.front()._url.c_str());
}
}
_stl_mutex.unlock();
}
msleep(1);
}
}
void DataDownloadWorker::replyFinished(QNetworkReply *reply) {
QNetworkRequest request = reply->request();
string url = request.url().toString().toStdString();
bool noError = false;
QNetworkReply::NetworkError error = reply->error();
switch ( error ) {
case QNetworkReply::ContentNotFoundError:
if ( isIgnorable(url) )
removeTask(url);
else
emit networkError(tr("Can't find remote file!"), url.c_str());
break;
case QNetworkReply::ConnectionRefusedError:
emit networkError(tr("Connection Refused!"), url.c_str());
break;
case QNetworkReply::TimeoutError:
emit networkError(tr("Timeout!"), url.c_str());
break;
case QNetworkReply::NoError:
noError = true;
break;
case QNetworkReply::HostNotFoundError:
emit networkError(tr("Host not found!"), url.c_str());
break;
default:
emit networkError(tr("Unknown Error!"), url.c_str());
break;
}
if ( noError && _task_paths.find(url) != _task_paths.end() ) {
DataReceiver *receiver = nullptr;
if ( _task_paths.find(url) != _task_paths.end() )
receiver = _task_paths.at(url);
QByteArray transferred;
if ( reply->hasRawHeader(QString("Content-Length").toUtf8() ) ) {
int total_size = atoi( reply->rawHeader("Content-Length") );
int size = total_size, size_read = 0;
const int buffer_size = 10240;
static char buffer[buffer_size];
memset(buffer, 0, buffer_size);
qint64 bytes_read = 0;
do {
bytes_read = reply->read( buffer, min(buffer_size, size) );
transferred.append(buffer, bytes_read);
size -= bytes_read;
size_read += bytes_read;
} while( size > 0 );
} else if ( reply->isFinished() ) {
QTextCodec *codec = QTextCodec::codecForName("utf8");
QString context = codec->toUnicode( reply->readAll() );
transferred.clear();
transferred.append(context);
}
if (receiver) {
receiver->receiveData(transferred);
NetFile *net_file = dynamic_cast<NetFile *>(receiver);
if ( net_file ) {
_stl_mutex.lock();
_task_paths.erase(url);
_task_queue.pop();
_stl_mutex.unlock();
emit NetFileDownLoaded(net_file);
}
}
}
disconnect( reply, SIGNAL(downloadProgress(qint64, qint64)),
this, SLOT(onTransferring(qint64, qint64)) );
reply->deleteLater();
_downloadingMutex.unlock();
}