C#_.net core 3.0自定义读取.csv文件数据_解决首行不是标题的问题_Linqtocsv改进

2022-07-12 14:28:58

　　linqtocsv文件有不太好的地方就是：无法设置标题的行数，默认首行就是标题，这不是很尴尬吗？并不是所有的csv文件严格写的首行是标题，下面全是数据，我接受的任务就是读取很多.csv报表数据，里面就有很多前几行是说明性内容，下面才是标题和数据。为了更好的解决这个问题，自己写吧...

　　本博客没有照搬linqtocsv全部源码，保留了主要功能，并对其优化，为我所用，哈哈...

　　下面是主要代码：

　　1-主文件CsvHelper：

　　这里在独自解析数据的时候，遇到了很多坑：

　　a-遇到数据含有分隔符的问题的解决办法，代码已经包含了

　　b-遇到了解析源文档数据时，未指定字符编码时，部分数据丢失导致csv文件个别行数据解析异常的问题，针对该问题，就是老老实实把读取文件时加了字符编码的参数进去，默认UTF-8。　　

using Microsoft.Extensions.Logging;
using PaymentAccountAPI.Helper;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;

namespace PaymentAccountAPI.CSV
{
    public class CsvHelper
    {
        /// <summary>
        /// 日志
        /// </summary>
        private ILogger _Logger { get; set; }

        public CsvHelper(ILogger<CsvHelper> logger)
        {
            this._Logger = logger;
        }

        public List<T> Read<T>(string filePath, CsvFileDescription fileDescription) where T : class, new()
        {
            List<T> tList = new List<T>(50 * 10000);

            T t = null;
            int currentRawIndex = 1;

            if (File.Exists(filePath))
            {
                using (StreamReader streamReader = new StreamReader(filePath, fileDescription.Encoding))
                {
                    Dictionary<int, FieldMapper> fieldMapperDic = FieldMapper.GetModelFieldMapper<T>().ToDictionary(m => m.CSVTitleIndex);
                    string rawValue = null;
                    string[] rawValueArray = null;
                    PropertyInfo propertyInfo = null;
                    string propertyValue = null;
                    bool rawReadEnd = false;

                    bool isExistSplitChart = false;
                    do
                    {
                        rawValue = streamReader.ReadLine();

                        //标题行
                        if (currentRawIndex > fileDescription.TitleRawIndex)
                        {
                            if (!string.IsNullOrEmpty(rawValue))
                            {
                                //替换字符串含有分隔符为{分隔符}，最后再替换回来
                                if (rawValue.Contains("\""))
                                {
                                    isExistSplitChart = true;

                                    int yhBeginIndex = 0;
                                    int yhEndIndex = 0;
                                    string yhText = null;
                                    do
                                    {
                                        yhBeginIndex = StringHelper.GetIndexOfStr(rawValue, "\"", 1);
                                        yhEndIndex = StringHelper.GetIndexOfStr(rawValue, "\"", 2);
                                        yhText = rawValue.Substring(yhBeginIndex, (yhEndIndex - yhBeginIndex + 1));
                                        string newYHText = yhText.Replace("\"", "").Replace(fileDescription.SeparatorChar.ToString(), "{分隔符}");
                                        rawValue = rawValue.Replace(yhText, newYHText);
                                    } while (rawValue.Contains("\""));
                                }

                                rawValueArray = rawValue.Split(fileDescription.SeparatorChar);

                                t = new T();
                                foreach (var fieldMapper in fieldMapperDic)
                                {
                                    propertyInfo = fieldMapper.Value.PropertyInfo;
                                    propertyValue = rawValueArray[fieldMapper.Key - 1];
                                    if (!string.IsNullOrEmpty(propertyValue))
                                    {
                                        try
                                        {
                                            if (isExistSplitChart && propertyValue.Contains("{分隔符}"))
                                            {
                                                propertyValue = propertyValue.Replace("{分隔符}", fileDescription.SeparatorChar.ToString());
                                            }

                                            TypeHelper.SetPropertyValue(t, propertyInfo.Name, propertyValue);
                                        }
                                        catch (Exception e)
                                        {
                                            this._Logger.LogWarning(e, $"第{currentRawIndex + 1}行数据{propertyValue}转换属性{propertyInfo.Name}-{propertyInfo.PropertyType.Name}失败！");
                                            continue;
                                        }
                                    }
                                }
                                tList.Add(t);
                            }
                            else
                            {
                                rawReadEnd = true;
                            }
                        }
                        currentRawIndex++;
                    } while (rawReadEnd == false);
                }
            }


            return tList;
        }

        public void WriteFile<T>(string path, List<T> tList, CsvFileDescription fileDescription) where T : class, new()
        {
            if (!string.IsNullOrEmpty(path))
            {
                string fileDirectoryPath = null;
                if (path.Contains("\\"))
                {
                    fileDirectoryPath = path.Substring(0, path.LastIndexOf(‘\\‘));
                }
                else
                {
                    fileDirectoryPath = path.Substring(0, path.LastIndexOf(‘/‘));
                }
                if (!Directory.Exists(fileDirectoryPath))
                {
                    Directory.CreateDirectory(fileDirectoryPath);
                }

                int dataCount = tList.Count;
                Dictionary<int, FieldMapper> fieldMapperDic = FieldMapper.GetModelFieldMapper<T>().ToDictionary(m => m.CSVTitleIndex);
                int titleCount = fieldMapperDic.Keys.Max();
                string[] rawValueArray = new string[titleCount];
                StringBuilder rawValueBuilder = new StringBuilder();
                string rawValue = null;
                T t = null;
                PropertyInfo propertyInfo = null;
                int currentRawIndex = 1;
                int tIndex = 0;

                using (StreamWriter streamWriter = new StreamWriter(path, false, fileDescription.Encoding))
                {
                    do
                    {
                        try
                        {
                            rawValue = "";

#if DEBUG
                            if (currentRawIndex % 10000 == 0)
                            {
                                this._Logger.LogInformation($"已写入文件：{path}，数据量：{currentRawIndex}");
                            }
#endif

                            if (currentRawIndex >= fileDescription.TitleRawIndex)
                            {
                                //清空数组数据
                                for (int i = 0; i < titleCount; i++)
                                {
                                    rawValueArray[i] = "";
                                }

                                if (currentRawIndex > fileDescription.TitleRawIndex)
                                {
                                    t = tList[tIndex];
                                    tIndex++;
                                }
                                foreach (var fieldMapperItem in fieldMapperDic)
                                {
                                    //写入标题行
                                    if (currentRawIndex == fileDescription.TitleRawIndex)
                                    {
                                        rawValueArray[fieldMapperItem.Key - 1] = fieldMapperItem.Value.CSVTitle;
                                    }
                                    //真正的数据从标题行下一行开始写
                                    else
                                    {
                                        propertyInfo = fieldMapperItem.Value.PropertyInfo;
                                        object propertyValue = propertyInfo.GetValue(t);
                                        string formatValue = null;
                                        if (propertyValue != null)
                                        {
                                            if (propertyInfo.PropertyType is IFormattable && !string.IsNullOrEmpty(fieldMapperItem.Value.OutputFormat))
                                            {
                                                formatValue = ((IFormattable)propertyValue).ToString(fieldMapperItem.Value.OutputFormat, null);
                                            }
                                            else
                                            {
                                                formatValue = propertyValue.ToString();
                                            }

                                            //如果属性值含有分隔符，则使用双引号包裹
                                            if (formatValue.Contains(fileDescription.SeparatorChar.ToString()))
                                            {
                                                formatValue = $"\"{formatValue}\"";
                                            }
                                            rawValueArray[fieldMapperItem.Key - 1] = formatValue;
                                        }
                                    }
                                }
                                rawValue = string.Join(fileDescription.SeparatorChar, rawValueArray);
                            }
                            rawValueBuilder.Append(rawValue + "\r\n");
                        }
                        catch (Exception e)
                        {
                            this._Logger.LogWarning(e, $"(异常)Excel第{currentRawIndex}行，数据列表第{tIndex + 1}个数据写入失败！rawValue：{rawValue}");
                            throw;
                        }

                        currentRawIndex++;
                    } while (tIndex < dataCount);
                    streamWriter.Write(rawValueBuilder.ToString());

                    streamWriter.Close();
                    streamWriter.Dispose();
                }
            }
        }

    }
}

　　2-CSV映射类特性：

using System;

namespace PaymentAccountAPI.CSV
{
    /// <summary>
    /// Csv文件类特性标记
    /// </summary>
    [System.AttributeUsage(System.AttributeTargets.Field | System.AttributeTargets.Property, AllowMultiple = false)]
    public class CsvColumnAttribute : System.Attribute
    {
        internal const int defaultTitleIndex = Int32.MaxValue;
        /// <summary>
        /// 标题
        /// </summary>
        public string Title { get; set; }
        /// <summary>
        /// 标题位置（从1开始）
        /// </summary>
        public int TitleIndex { get; set; }
        /// <summary>
        /// 字符输出格式（数字和日期类型需要）
        /// </summary>
        public string OutputFormat { get; set; }

        public CsvColumnAttribute()
        {
            Title = "";
            TitleIndex = defaultTitleIndex;
            OutputFormat = "";
        }

        public CsvColumnAttribute(string title, int titleIndex, string outputFormat)
        {
            Title = title;
            TitleIndex = titleIndex;
            OutputFormat = outputFormat;
        }
    }
}

　　3-CSV文件描述信息类：

using System.Text;

namespace PaymentAccountAPI.CSV
{
    public class CsvFileDescription
    {
        public CsvFileDescription() : this(1)
        {
        }
        public CsvFileDescription(int titleRawIndex) : this(‘,‘, titleRawIndex, Encoding.UTF8)
        {
        }
        public CsvFileDescription(char separatorChar, int titleRawIndex, Encoding encoding)
        {
            this.SeparatorChar = separatorChar;
            this.TitleRawIndex = titleRawIndex;
            this.Encoding = encoding;
        }

        /// <summary>
        /// CSV文件字符编码
        /// </summary>
        public Encoding Encoding { get; set; }

        /// <summary>
        /// 分隔符（默认为（,）,也可以是其他分隔符如(\t)）
        /// </summary>
        public char SeparatorChar { get; set; }
        /// <summary>
        /// 标题所在行位置（默认为1，没有标题填0）
        /// </summary>
        public int TitleRawIndex { get; set; }

    }
}

　　4-映射类获取关系帮助类：

using System.Collections.Generic;
using System.Linq;
using System.Reflection;

namespace PaymentAccountAPI.CSV
{
    /// <summary>
    /// 字段映射类
    /// </summary>
    public class FieldMapper
    {
        /// <summary>
        /// 属性信息
        /// </summary>
        public PropertyInfo PropertyInfo { get; set; }
        /// <summary>
        /// 标题
        /// </summary>
        public string CSVTitle { get; set; }
        /// <summary>
        /// 标题下标位置
        /// </summary>
        public int CSVTitleIndex { get; set; }
        /// <summary>
        /// 字符输出格式（数字和日期类型需要）
        /// </summary>
        public string OutputFormat { get; set; }

        public static List<FieldMapper> GetModelFieldMapper<T>()
        {
            List<FieldMapper> fieldMapperList = new List<FieldMapper>(100);

            List<PropertyInfo> tPropertyInfoList = typeof(T).GetProperties().ToList();
            CsvColumnAttribute csvColumnAttribute = null;
            foreach (var tPropertyInfo in tPropertyInfoList)
            {
                csvColumnAttribute = (CsvColumnAttribute)tPropertyInfo.GetCustomAttribute(typeof(CsvColumnAttribute));
                if (csvColumnAttribute != null)
                {
                    fieldMapperList.Add(new FieldMapper
                    {
                        PropertyInfo = tPropertyInfo,
                        CSVTitle = csvColumnAttribute.Title,
                        CSVTitleIndex = csvColumnAttribute.TitleIndex,
                        OutputFormat = csvColumnAttribute.OutputFormat
                    });
                }
            }
            return fieldMapperList;
        }

    }

}

　　5-其他扩展类：

namespace PaymentAccountAPI.Helper
{
    public class StringHelper
    {
        /// <summary>
        /// 获取字符串中第strPosition个位置的str的下标
        /// </summary>
        /// <param name="text"></param>
        /// <param name="str"></param>
        /// <param name="strPosition"></param>
        /// <returns></returns>
        public static int GetIndexOfStr(string text, string str, int strPosition)
        {
            int strIndex = -1;

            int currentPosition = 0;
            if (!string.IsNullOrEmpty(text) && !string.IsNullOrEmpty(str) && strPosition >= 1)
            {
                do
                {
                    currentPosition++;
                    if (strIndex == -1)
                    {
                        strIndex = text.IndexOf(str);
                    }
                    else
                    {
                        strIndex = text.IndexOf(str, strIndex + 1);
                    }
                } while (currentPosition < strPosition);
            }

            return strIndex;
        }
    }
}

　　最后就是将CsvHelper注入到单例中，就可以使用了...

C#_.net core 3.0自定义读取.csv文件数据_解决首行不是标题的问题_Linqtocsv改进

码农公寓

相关文章